x86, asm: Introduce and use percpu_inc()

... generating slightly smaller code.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
LKML-Reference: <4BCF261F020000780003B33C@vpn.id2.novell.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 0f8576427cfefff4ada5dfdaac118e01428d8997..aeab29aee617240fbf479d2945572879be4525ec 100644 (file)
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,7 +35,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 
 #define __ARCH_IRQ_STAT
 
-#define inc_irq_stat(member)   percpu_add(irq_stat.member, 1)
+#define inc_irq_stat(member)   percpu_inc(irq_stat.member)
 
 #define local_softirq_pending()        percpu_read(irq_stat.__softirq_pending)
 

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 66a272dfd8b8aa0d5fcb788ace9bec17da31e236..0ec6d12d84e68b377909da90f18ec0e3488f4526 100644 (file)
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -190,6 +190,29 @@ do {                                                                       \
        pfo_ret__;                                      \
 })
 
+#define percpu_unary_op(op, var)                       \
+({                                                     \
+       switch (sizeof(var)) {                          \
+       case 1:                                         \
+               asm(op "b "__percpu_arg(0)              \
+                   : "+m" (var));                      \
+               break;                                  \
+       case 2:                                         \
+               asm(op "w "__percpu_arg(0)              \
+                   : "+m" (var));                      \
+               break;                                  \
+       case 4:                                         \
+               asm(op "l "__percpu_arg(0)              \
+                   : "+m" (var));                      \
+               break;                                  \
+       case 8:                                         \
+               asm(op "q "__percpu_arg(0)              \
+                   : "+m" (var));                      \
+               break;                                  \
+       default: __bad_percpu_size();                   \
+       }                                               \
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -207,6 +230,7 @@ do {                                                                        \
 #define percpu_and(var, val)           percpu_to_op("and", var, val)
 #define percpu_or(var, val)            percpu_to_op("or", var, val)
 #define percpu_xor(var, val)           percpu_to_op("xor", var, val)
+#define percpu_inc(var)                percpu_unary_op("inc", var)
 
 #define __this_cpu_read_1(pcp)         percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_read_2(pcp)         percpu_from_op("mov", (pcp), "m"(pcp))

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 8a6f0afa767ec804c0a1ffa2f6bfee6e87168cb1..7a355ddcc64b98707fef017c084937b82219aa89 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -539,7 +539,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
        struct mce m;
        int i;
 
-       __get_cpu_var(mce_poll_count)++;
+       percpu_inc(mce_poll_count);
 
        mce_setup(&m);
 
@@ -934,7 +934,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 
        atomic_inc(&mce_entry);
 
-       __get_cpu_var(mce_exception_count)++;
+       percpu_inc(mce_exception_count);
 
        if (notify_die(DIE_NMI, "machine check", regs, error_code,
                           18, SIGKILL) == NOTIFY_STOP)