]> git.karo-electronics.de Git - linux-beck.git/commitdiff
arch, locking: Ciao arch_mutex_cpu_relax()
authorDavidlohr Bueso <davidlohr@hp.com>
Sun, 29 Jun 2014 22:09:33 +0000 (15:09 -0700)
committerIngo Molnar <mingo@kernel.org>
Thu, 17 Jul 2014 10:32:47 +0000 (12:32 +0200)
The arch_mutex_cpu_relax() function, introduced by 34b133f, is
hacky and ugly. It was added a few years ago to address the fact
that common cpu_relax() calls include yielding on s390, and thus
impact the optimistic spinning functionality of mutexes. Nowadays
we use this function well beyond mutexes: rwsem, qrwlock, mcs and
lockref. Since the macro that defines the call is in the mutex header,
any users must include mutex.h and the naming is misleading as well.

This patch (i) renames the call to cpu_relax_lowlatency  ("relax, but
only if you can do it with very low latency") and (ii) defines it in
each arch's asm/processor.h local header, just like for regular cpu_relax
functions. On all archs, except s390, cpu_relax_lowlatency is simply cpu_relax,
and thus we can take it out of mutex.h. While this can seem redundant,
I believe it is a good choice as it allows us to move out arch specific
logic from generic locking primitives and enables future(?) archs to
transparently define it, similarly to System Z.

Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Anton Blanchard <anton@samba.org>
Cc: Aurelien Jacquiot <a-jacquiot@ti.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Bharat Bhushan <r65777@freescale.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Chen Liqin <liqin.linux@gmail.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Chris Zankel <chris@zankel.net>
Cc: David Howells <dhowells@redhat.com>
Cc: David S. Miller <davem@davemloft.net>
Cc: Deepthi Dharwar <deepthi@linux.vnet.ibm.com>
Cc: Dominik Dingel <dingel@linux.vnet.ibm.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Guan Xuetao <gxt@mprc.pku.edu.cn>
Cc: Haavard Skinnemoen <hskinnemoen@gmail.com>
Cc: Hans-Christian Egtvedt <egtvedt@samfundet.no>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: James E.J. Bottomley <jejb@parisc-linux.org>
Cc: James Hogan <james.hogan@imgtec.com>
Cc: Jason Wang <jasowang@redhat.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: Joe Perches <joe@perches.com>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Joseph Myers <joseph@codesourcery.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: Lennox Wu <lennox.wu@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mark Salter <msalter@redhat.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Max Filippov <jcmvbkbc@gmail.com>
Cc: Michael Neuling <mikey@neuling.org>
Cc: Michal Simek <monstr@monstr.eu>
Cc: Mikael Starvik <starvik@axis.com>
Cc: Nicolas Pitre <nico@linaro.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Paul Burton <paul.burton@imgtec.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Qais Yousef <qais.yousef@imgtec.com>
Cc: Qiaowei Ren <qiaowei.ren@intel.com>
Cc: Rafael Wysocki <rafael.j.wysocki@intel.com>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Richard Henderson <rth@twiddle.net>
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Steven Miao <realmz6@gmail.com>
Cc: Steven Rostedt <srostedt@redhat.com>
Cc: Stratos Karafotis <stratosk@semaphore.gr>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Vasily Kulikov <segoon@openwall.com>
Cc: Vineet Gupta <vgupta@synopsys.com>
Cc: Vineet Gupta <Vineet.Gupta1@synopsys.com>
Cc: Waiman Long <Waiman.Long@hp.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Wolfram Sang <wsa@the-dreams.de>
Cc: adi-buildroot-devel@lists.sourceforge.net
Cc: linux390@de.ibm.com
Cc: linux-alpha@vger.kernel.org
Cc: linux-am33-list@redhat.com
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-c6x-dev@linux-c6x.org
Cc: linux-cris-kernel@axis.com
Cc: linux-hexagon@vger.kernel.org
Cc: linux-ia64@vger.kernel.org
Cc: linux@lists.openrisc.net
Cc: linux-m32r-ja@ml.linux-m32r.org
Cc: linux-m32r@ml.linux-m32r.org
Cc: linux-m68k@lists.linux-m68k.org
Cc: linux-metag@vger.kernel.org
Cc: linux-mips@linux-mips.org
Cc: linux-parisc@vger.kernel.org
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-s390@vger.kernel.org
Cc: linux-sh@vger.kernel.org
Cc: linux-xtensa@linux-xtensa.org
Cc: sparclinux@vger.kernel.org
Link: http://lkml.kernel.org/r/1404079773.2619.4.camel@buesod1.americas.hpqcorp.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
36 files changed:
arch/alpha/include/asm/processor.h
arch/arc/include/asm/processor.h
arch/arm/include/asm/processor.h
arch/arm64/include/asm/processor.h
arch/avr32/include/asm/processor.h
arch/blackfin/include/asm/processor.h
arch/c6x/include/asm/processor.h
arch/cris/include/asm/processor.h
arch/hexagon/include/asm/processor.h
arch/ia64/include/asm/processor.h
arch/m32r/include/asm/processor.h
arch/m68k/include/asm/processor.h
arch/metag/include/asm/processor.h
arch/microblaze/include/asm/processor.h
arch/mips/include/asm/processor.h
arch/mn10300/include/asm/processor.h
arch/openrisc/include/asm/processor.h
arch/parisc/include/asm/processor.h
arch/powerpc/include/asm/processor.h
arch/s390/include/asm/processor.h
arch/score/include/asm/processor.h
arch/sh/include/asm/processor.h
arch/sparc/include/asm/processor_32.h
arch/sparc/include/asm/processor_64.h
arch/tile/include/asm/processor.h
arch/unicore32/include/asm/processor.h
arch/x86/include/asm/processor.h
arch/x86/um/asm/processor.h
arch/xtensa/include/asm/processor.h
include/linux/mutex.h
kernel/locking/mcs_spinlock.c
kernel/locking/mcs_spinlock.h
kernel/locking/mutex.c
kernel/locking/qrwlock.c
kernel/locking/rwsem-xadd.c
lib/lockref.c

index 6cb7fe85c4b567adad3516a74c5b862a32a50a5e..b4cf036903942cb984ede0e5ba0b880526fa46c2 100644 (file)
@@ -57,6 +57,7 @@ unsigned long get_wchan(struct task_struct *p);
   ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp)
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 #define ARCH_HAS_PREFETCH
 #define ARCH_HAS_PREFETCHW
index d99f9b37cd1540a14004e56a46d0e7be0584891b..82588f3ba77f6b940fd8aa607af34d049614a993 100644 (file)
@@ -62,6 +62,8 @@ unsigned long thread_saved_pc(struct task_struct *t);
 #define cpu_relax()    do { } while (0)
 #endif
 
+#define cpu_relax_lowlatency() cpu_relax()
+
 #define copy_segments(tsk, mm)      do { } while (0)
 #define release_segments(mm)        do { } while (0)
 
index c3d5fc124a054c6309ffacdb2845ff22fd5bfa56..8a1e8e995daec45278d088faa3ada7f1d6dbb957 100644 (file)
@@ -82,6 +82,8 @@ unsigned long get_wchan(struct task_struct *p);
 #define cpu_relax()                    barrier()
 #endif
 
+#define cpu_relax_lowlatency()                cpu_relax()
+
 #define task_pt_regs(p) \
        ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
 
index 34de2a8f7d9340c8e1ecb7cd78be568ad81147f9..4610b0daf1bf0955818259e5a44270b5fef13dab 100644 (file)
@@ -129,6 +129,7 @@ extern void release_thread(struct task_struct *);
 unsigned long get_wchan(struct task_struct *p);
 
 #define cpu_relax()                    barrier()
+#define cpu_relax_lowlatency()                cpu_relax()
 
 /* Thread switching */
 extern struct task_struct *cpu_switch_to(struct task_struct *prev,
index 972adcc1e8f40179d06a4062d2adf63416f36452..941593c7d9f3c12d841283e3330d2ea7fbef4cdb 100644 (file)
@@ -92,6 +92,7 @@ extern struct avr32_cpuinfo boot_cpu_data;
 #define TASK_UNMAPPED_BASE     (PAGE_ALIGN(TASK_SIZE / 3))
 
 #define cpu_relax()            barrier()
+#define cpu_relax_lowlatency()        cpu_relax()
 #define cpu_sync_pipeline()    asm volatile("sub pc, -2" : : : "memory")
 
 struct cpu_context {
index d0e72e9475a6ba3627fcad60ff3f6e3f34cdc15d..7acd46653df3680d5208020945365063c57891af 100644 (file)
@@ -99,7 +99,7 @@ unsigned long get_wchan(struct task_struct *p);
 #define        KSTK_ESP(tsk)   ((tsk) == current ? rdusp() : (tsk)->thread.usp)
 
 #define cpu_relax()            smp_mb()
-
+#define cpu_relax_lowlatency() cpu_relax()
 
 /* Get the Silicon Revision of the chip */
 static inline uint32_t __pure bfin_revid(void)
index b9eb3da7f278dac858bfb2191094f9fc837ac5c8..f2ef31be2f8b0c1f196c1352eace1511590a2360 100644 (file)
@@ -121,6 +121,7 @@ extern unsigned long get_wchan(struct task_struct *p);
 #define KSTK_ESP(task) (task_pt_regs(task)->sp)
 
 #define cpu_relax()            do { } while (0)
+#define cpu_relax_lowlatency()        cpu_relax()
 
 extern const struct seq_operations cpuinfo_op;
 
index 15b815df29c165809c4e6e229ede6a077d9e8e71..862126b58116b03e3e1c7793883999c3ab8236a0 100644 (file)
@@ -63,6 +63,7 @@ static inline void release_thread(struct task_struct *dead_task)
 #define init_stack      (init_thread_union.stack)
 
 #define cpu_relax()     barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 void default_idle(void);
 
index 45a825402f634ee4b650a08a60efc5c60d50e261..d8501137c8d022db81fb14676fc40f40ede0d8fa 100644 (file)
@@ -56,6 +56,7 @@ struct thread_struct {
 }
 
 #define cpu_relax() __vmyield()
+#define cpu_relax_lowlatency() cpu_relax()
 
 /*
  * Decides where the kernel will search for a free chunk of vm space during
index efd1b927ccb7ebe57934a566f8a204cdbf8733b1..c7367130ab14c07c1f643ac867154e01efd80114 100644 (file)
@@ -548,6 +548,7 @@ ia64_eoi (void)
 }
 
 #define cpu_relax()    ia64_hint(ia64_hint_pause)
+#define cpu_relax_lowlatency() cpu_relax()
 
 static inline int
 ia64_get_irr(unsigned int vector)
index 5767367550c69637a0b51c7a037b7a5e0c4fbfb4..9f8fd9bef70f58d8c8a2368d777d59e3f9d1dd6d 100644 (file)
@@ -133,5 +133,6 @@ unsigned long get_wchan(struct task_struct *p);
 #define KSTK_ESP(tsk)  ((tsk)->thread.sp)
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 #endif /* _ASM_M32R_PROCESSOR_H */
index b0768a657920f63c51522f2ddb0ef22da0c8dbc3..20dda1d4b86014975a68b1abde20177fa104482b 100644 (file)
@@ -176,5 +176,6 @@ unsigned long get_wchan(struct task_struct *p);
 #define task_pt_regs(tsk)      ((struct pt_regs *) ((tsk)->thread.esp0))
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 #endif
index a8a37477c66e22a8e3c29ba2402ff6559a35bf27..881071c0794221475308604464cd7bd441b95945 100644 (file)
@@ -155,6 +155,7 @@ unsigned long get_wchan(struct task_struct *p);
 #define user_stack_pointer(regs)        ((regs)->ctx.AX[0].U0)
 
 #define cpu_relax()     barrier()
+#define cpu_relax_lowlatency()  cpu_relax()
 
 extern void setup_priv(void);
 
index 9d31b057c3553da4d7f697179ef0c92a4223d5df..497a988d79c2346126de754c5f1fe246d50a7a4c 100644 (file)
@@ -22,6 +22,7 @@
 extern const struct seq_operations cpuinfo_op;
 
 # define cpu_relax()           barrier()
+# define cpu_relax_lowlatency()        cpu_relax()
 
 #define task_pt_regs(tsk) \
                (((struct pt_regs *)(THREAD_SIZE + task_stack_page(tsk))) - 1)
index ad70cba8daffaeaeb7f62a7a47308dc1597a8be4..d5098bc554f4bb675f8ccf7ba56eb9c4e342f477 100644 (file)
@@ -367,6 +367,7 @@ unsigned long get_wchan(struct task_struct *p);
 #define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status)
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 /*
  * Return_address is a replacement for __builtin_return_address(count)
index 8b80b19d0c8a4fbfe7406f12141ae6d12040fd35..769d5ed8e9923c7f4da6b79eef9a53c5a888e8be 100644 (file)
@@ -68,7 +68,9 @@ extern struct mn10300_cpuinfo cpu_data[];
 extern void identify_cpu(struct mn10300_cpuinfo *);
 extern void print_cpu_info(struct mn10300_cpuinfo *);
 extern void dodgy_tsc(void);
+
 #define cpu_relax() barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 /*
  * User space process size: 1.75GB (default).
index cab746fa9e8731a6599802a79dfcc56e044d97d4..4d235e3d253458f2d202d253a7667f1958e8612d 100644 (file)
@@ -101,6 +101,7 @@ extern unsigned long thread_saved_pc(struct task_struct *t);
 #define init_stack      (init_thread_union.stack)
 
 #define cpu_relax()     barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 #endif /* __ASSEMBLY__ */
 #endif /* __ASM_OPENRISC_PROCESSOR_H */
index d951c9681ab316aa2be88cd65b4d18ae62f958cb..689a8ade3606da284dc29325b35d7f42d06f37ad 100644 (file)
@@ -338,6 +338,7 @@ extern unsigned long get_wchan(struct task_struct *p);
 #define KSTK_ESP(tsk)  ((tsk)->thread.regs.gr[30])
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 /* Used as a macro to identify the combined VIPT/PIPT cached
  * CPUs which require a guarantee of coherency (no inequivalent
index 6d59072e13a7c9244ed3c8716389a433dddb73cf..dda7ac4c80bdd807c76cb9af3e6e684900b61f60 100644 (file)
@@ -400,6 +400,8 @@ static inline unsigned long __pack_fe01(unsigned int fpmode)
 #define cpu_relax()    barrier()
 #endif
 
+#define cpu_relax_lowlatency() cpu_relax()
+
 /* Check that a certain kernel stack pointer is valid in task_struct p */
 int validate_sp(unsigned long sp, struct task_struct *p,
                        unsigned long nbytes);
index 6f02d452bbee3741ac164ed3f1f16fe0970d2fe8..e568fc8a725062f707b7875c7c4010a21bd0fcfa 100644 (file)
@@ -217,7 +217,7 @@ static inline void cpu_relax(void)
        barrier();
 }
 
-#define arch_mutex_cpu_relax()  barrier()
+#define cpu_relax_lowlatency()  barrier()
 
 static inline void psw_set_key(unsigned int key)
 {
index d9a922d8711b2155a2225699e5958236dbb70adb..851f441991d2764315057654ef988db57c659dac 100644 (file)
@@ -24,6 +24,7 @@ extern unsigned long get_wchan(struct task_struct *p);
 #define current_text_addr() ({ __label__ _l; _l: &&_l; })
 
 #define cpu_relax()            barrier()
+#define cpu_relax_lowlatency()        cpu_relax()
 #define release_thread(thread) do {} while (0)
 
 /*
index 5448f9bbf4ab11394601f938e7a791e40c2b1c9f..1506897648aa17ce0d594b2d1068fc7988a70095 100644 (file)
@@ -97,6 +97,7 @@ extern struct sh_cpuinfo cpu_data[];
 
 #define cpu_sleep()    __asm__ __volatile__ ("sleep" : : : "memory")
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 void default_idle(void);
 void stop_this_cpu(void *);
index a564817bbc2e1de03ba60614ea4d2220ab00b4d2..812fd08f3e6253ecb490ff149894bafae79bbc26 100644 (file)
@@ -119,6 +119,8 @@ extern struct task_struct *last_task_used_math;
 int do_mathemu(struct pt_regs *regs, struct task_struct *fpt);
 
 #define cpu_relax()    barrier()
+#define cpu_relax_lowlatency() cpu_relax()
+
 extern void (*sparc_idle)(void);
 
 #endif
index 7028fe1a7c042a9b2cca56da21a9b7e48339a248..6924bdefe148451c584f0a6b183b8a23b92bae05 100644 (file)
@@ -216,6 +216,7 @@ unsigned long get_wchan(struct task_struct *task);
                                     "nop\n\t"                          \
                                     ".previous"                        \
                                     ::: "memory")
+#define cpu_relax_lowlatency() cpu_relax()
 
 /* Prefetch support.  This is tuned for UltraSPARC-III and later.
  * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
index 42323636c459f3577e6053bad26c36c4adb1535b..dd4f9f17e30aa8c5887607c5f5f933f3ab00666e 100644 (file)
@@ -266,6 +266,8 @@ static inline void cpu_relax(void)
        barrier();
 }
 
+#define cpu_relax_lowlatency() cpu_relax()
+
 /* Info on this processor (see fs/proc/cpuinfo.c) */
 struct seq_operations;
 extern const struct seq_operations cpuinfo_op;
index 4eaa4216766772055849a3ffdb1b6e20d30edecb..8d21b7adf26bc3709ba865b6f29db2a18436f0a0 100644 (file)
@@ -71,6 +71,7 @@ extern void release_thread(struct task_struct *);
 unsigned long get_wchan(struct task_struct *p);
 
 #define cpu_relax()                    barrier()
+#define cpu_relax_lowlatency()                cpu_relax()
 
 #define task_pt_regs(p) \
        ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
index a4ea02351f4d02bee5a43e78a9013b77ec94ea48..32cc237f8e2062edf2eb8cc26d0a738dd1cd8822 100644 (file)
@@ -696,6 +696,8 @@ static inline void cpu_relax(void)
        rep_nop();
 }
 
+#define cpu_relax_lowlatency() cpu_relax()
+
 /* Stop speculative execution and prefetching of modified code. */
 static inline void sync_core(void)
 {
index 04f82e020f2bdafbec85118e99ff73aed925ba97..2a206d2b14ab55a325236b9a871897d84b3367f4 100644 (file)
@@ -25,7 +25,8 @@ static inline void rep_nop(void)
        __asm__ __volatile__("rep;nop": : :"memory");
 }
 
-#define cpu_relax()    rep_nop()
+#define cpu_relax()            rep_nop()
+#define cpu_relax_lowlatency() cpu_relax()
 
 #include <asm/processor-generic.h>
 
index abb59708a3b75bf6ac8398be37e4fc90c349dff0..b61bdf0eea254457c12013696106aa260efaba76 100644 (file)
@@ -182,6 +182,7 @@ extern unsigned long get_wchan(struct task_struct *p);
 #define KSTK_ESP(tsk)          (task_pt_regs(tsk)->areg[1])
 
 #define cpu_relax()  barrier()
+#define cpu_relax_lowlatency() cpu_relax()
 
 /* Special register access. */
 
index 42aa9b9ecd5f8ded624389a476952c7b5ae9a2a8..8d5535c58cc28578714c99eb6a4c16bbdd5726a9 100644 (file)
@@ -176,8 +176,4 @@ extern void mutex_unlock(struct mutex *lock);
 
 extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
 
-#ifndef arch_mutex_cpu_relax
-# define arch_mutex_cpu_relax() cpu_relax()
-#endif
-
 #endif /* __LINUX_MUTEX_H */
index be9ee1559fca5243bd2c6689117a8dde6ee50fb0..9887a905a7626278d8ccf6da8176c447e2aad988 100644 (file)
@@ -1,6 +1,4 @@
-
 #include <linux/percpu.h>
-#include <linux/mutex.h>
 #include <linux/sched.h>
 #include "mcs_spinlock.h"
 
@@ -79,7 +77,7 @@ osq_wait_next(struct optimistic_spin_queue *lock,
                                break;
                }
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
 
        return next;
@@ -120,7 +118,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
                if (need_resched())
                        goto unqueue;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        return true;
 
@@ -146,7 +144,7 @@ unqueue:
                if (smp_load_acquire(&node->locked))
                        return true;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
 
                /*
                 * Or we race against a concurrent unqueue()'s step-B, in which
index 74356dc0ce298c8b9092edc4c48e7a6a62af3830..23e89c5930e9414553f99dfa30ef4d154106d13b 100644 (file)
@@ -27,7 +27,7 @@ struct mcs_spinlock {
 #define arch_mcs_spin_lock_contended(l)                                        \
 do {                                                                   \
        while (!(smp_load_acquire(l)))                                  \
-               arch_mutex_cpu_relax();                                 \
+               cpu_relax_lowlatency();                                 \
 } while (0)
 #endif
 
@@ -104,7 +104,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
                        return;
                /* Wait until the next pointer is set */
                while (!(next = ACCESS_ONCE(node->next)))
-                       arch_mutex_cpu_relax();
+                       cpu_relax_lowlatency();
        }
 
        /* Pass lock to next waiter. */
index d3100521388c04eb4cb78b7dd99f731542936eb1..ae712b25e4922517a5fc13949d09f2c36294dadb 100644 (file)
@@ -146,7 +146,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
                if (need_resched())
                        break;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        rcu_read_unlock();
 
@@ -464,7 +464,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 * memory barriers as we'll eventually observe the right
                 * values at the cost of a few extra spins.
                 */
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        osq_unlock(&lock->osq);
 slowpath:
index fb5b8ac411a58e6c8466662207dabdf3aac75a1d..f956ede7f90df0b31210df87843f1bafc2ddc7cb 100644 (file)
@@ -20,7 +20,6 @@
 #include <linux/cpumask.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
-#include <linux/mutex.h>
 #include <asm/qrwlock.h>
 
 /**
@@ -35,7 +34,7 @@ static __always_inline void
 rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
 {
        while ((cnts & _QW_WMASK) == _QW_LOCKED) {
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
                cnts = smp_load_acquire((u32 *)&lock->cnts);
        }
 }
@@ -75,7 +74,7 @@ void queue_read_lock_slowpath(struct qrwlock *lock)
         * to make sure that the write lock isn't taken.
         */
        while (atomic_read(&lock->cnts) & _QW_WMASK)
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
 
        cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
        rspin_until_writer_unlock(lock, cnts);
@@ -114,7 +113,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock)
                                    cnts | _QW_WAITING) == cnts))
                        break;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
 
        /* When no more readers, set the locked flag */
@@ -125,7 +124,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock)
                                    _QW_LOCKED) == _QW_WAITING))
                        break;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
 unlock:
        arch_spin_unlock(&lock->lock);
index a2391ac135c8d4ffb83aabee5870d89287ebf64f..d6203faf2eb1f15522b720605bab13f6008fab4d 100644 (file)
@@ -329,7 +329,7 @@ bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)
                if (need_resched())
                        break;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        rcu_read_unlock();
 
@@ -381,7 +381,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
                 * memory barriers as we'll eventually observe the right
                 * values at the cost of a few extra spins.
                 */
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        osq_unlock(&sem->osq);
 done:
index f07a40d33871e2b9414ea40b53ee154108932d4e..d2233de9a86e564f297e01305ddbc41392b3d2ee 100644 (file)
@@ -1,6 +1,5 @@
 #include <linux/export.h>
 #include <linux/lockref.h>
-#include <linux/mutex.h>
 
 #if USE_CMPXCHG_LOCKREF
 
@@ -29,7 +28,7 @@
                if (likely(old.lock_count == prev.lock_count)) {                \
                        SUCCESS;                                                \
                }                                                               \
-               arch_mutex_cpu_relax();                                         \
+               cpu_relax_lowlatency();                                         \
        }                                                                       \
 } while (0)