Merge branch 'x86-asmlinkage-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[karo-tx-linux.git] / kernel / locking / mutex.c

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index adbc0d0f314b0049540d0eaf97afc4387513b4a3..bc73d33c6760e174fd1bb2c8319c0faf5abc221f 100644 (file)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -25,6 +25,7 @@
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
 #include <linux/debug_locks.h>
+#include "mcs_spinlock.h"
 
 /*
  * In the DEBUG case we are using the "NULL fastpath" for mutexes,
@@ -33,6 +34,13 @@
 #ifdef CONFIG_DEBUG_MUTEXES
 # include "mutex-debug.h"
 # include <asm-generic/mutex-null.h>
+/*
+ * Must be 0 for the debug case so we do not do the unlock outside of the
+ * wait_lock region. debug_mutex_unlock() will do the actual unlock in this
+ * case.
+ */
+# undef __mutex_slowpath_needs_to_unlock
+# define  __mutex_slowpath_needs_to_unlock()   0
 #else
 # include "mutex.h"
 # include <asm/mutex.h>
@@ -52,7 +60,7 @@ __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
        INIT_LIST_HEAD(&lock->wait_list);
        mutex_clear_owner(lock);
 #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
-       lock->spin_mlock = NULL;
+       lock->osq = NULL;
 #endif
 
        debug_mutex_init(lock, name, key);
@@ -110,54 +118,7 @@ EXPORT_SYMBOL(mutex_lock);
  * more or less simultaneously, the spinners need to acquire a MCS lock
  * first before spinning on the owner field.
  *
- * We don't inline mspin_lock() so that perf can correctly account for the
- * time spent in this lock function.
  */
-struct mspin_node {
-       struct mspin_node *next ;
-       int               locked;       /* 1 if lock acquired */
-};
-#define        MLOCK(mutex)    ((struct mspin_node **)&((mutex)->spin_mlock))
-
-static noinline
-void mspin_lock(struct mspin_node **lock, struct mspin_node *node)
-{
-       struct mspin_node *prev;
-
-       /* Init node */
-       node->locked = 0;
-       node->next   = NULL;
-
-       prev = xchg(lock, node);
-       if (likely(prev == NULL)) {
-               /* Lock acquired */
-               node->locked = 1;
-               return;
-       }
-       ACCESS_ONCE(prev->next) = node;
-       smp_wmb();
-       /* Wait until the lock holder passes the lock down */
-       while (!ACCESS_ONCE(node->locked))
-               arch_mutex_cpu_relax();
-}
-
-static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
-{
-       struct mspin_node *next = ACCESS_ONCE(node->next);
-
-       if (likely(!next)) {
-               /*
-                * Release the lock by setting it to NULL
-                */
-               if (cmpxchg(lock, node, NULL) == node)
-                       return;
-               /* Wait until the next pointer is set */
-               while (!(next = ACCESS_ONCE(node->next)))
-                       arch_mutex_cpu_relax();
-       }
-       ACCESS_ONCE(next->locked) = 1;
-       smp_wmb();
-}
 
 /*
  * Mutex spinning code migrated from kernel/sched/core.c
@@ -211,6 +172,9 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)
        struct task_struct *owner;
        int retval = 1;
 
+       if (need_resched())
+               return 0;
+
        rcu_read_lock();
        owner = ACCESS_ONCE(lock->owner);
        if (owner)
@@ -446,9 +410,11 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
        if (!mutex_can_spin_on_owner(lock))
                goto slowpath;
 
+       if (!osq_lock(&lock->osq))
+               goto slowpath;
+
        for (;;) {
                struct task_struct *owner;
-               struct mspin_node  node;
 
                if (use_ww_ctx && ww_ctx->acquired > 0) {
                        struct ww_mutex *ww;
@@ -463,19 +429,16 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                         * performed the optimistic spinning cannot be done.
                         */
                        if (ACCESS_ONCE(ww->ctx))
-                               goto slowpath;
+                               break;
                }
 
                /*
                 * If there's an owner, wait for it to either
                 * release the lock or go to sleep.
                 */
-               mspin_lock(MLOCK(lock), &node);
                owner = ACCESS_ONCE(lock->owner);
-               if (owner && !mutex_spin_on_owner(lock, owner)) {
-                       mspin_unlock(MLOCK(lock), &node);
-                       goto slowpath;
-               }
+               if (owner && !mutex_spin_on_owner(lock, owner))
+                       break;
 
                if ((atomic_read(&lock->count) == 1) &&
                    (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
@@ -488,11 +451,10 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                        }
 
                        mutex_set_owner(lock);
-                       mspin_unlock(MLOCK(lock), &node);
+                       osq_unlock(&lock->osq);
                        preempt_enable();
                        return 0;
                }
-               mspin_unlock(MLOCK(lock), &node);
 
                /*
                 * When there's no owner, we might have preempted between the
@@ -501,7 +463,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 * the owner complete.
                 */
                if (!owner && (need_resched() || rt_task(task)))
-                       goto slowpath;
+                       break;
 
                /*
                 * The cpu_relax() call is a compiler barrier which forces
@@ -511,7 +473,15 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 */
                arch_mutex_cpu_relax();
        }
+       osq_unlock(&lock->osq);
 slowpath:
+       /*
+        * If we fell out of the spin path because of need_resched(),
+        * reschedule now, before we try-lock the mutex. This avoids getting
+        * scheduled out right after we obtained the mutex.
+        */
+       if (need_resched())
+               schedule_preempt_disabled();
 #endif
        spin_lock_mutex(&lock->wait_lock, flags);
 
@@ -717,10 +687,6 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
        struct mutex *lock = container_of(lock_count, struct mutex, count);
        unsigned long flags;
 
-       spin_lock_mutex(&lock->wait_lock, flags);
-       mutex_release(&lock->dep_map, nested, _RET_IP_);
-       debug_mutex_unlock(lock);
-
        /*
         * some architectures leave the lock unlocked in the fastpath failure
         * case, others need to leave it locked. In the later case we have to
@@ -729,6 +695,10 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
        if (__mutex_slowpath_needs_to_unlock())
                atomic_set(&lock->count, 1);
 
+       spin_lock_mutex(&lock->wait_lock, flags);
+       mutex_release(&lock->dep_map, nested, _RET_IP_);
+       debug_mutex_unlock(lock);
+
        if (!list_empty(&lock->wait_list)) {
                /* get the first entry from the wait-list: */
                struct mutex_waiter *waiter =