futex: Sanitize cmpxchg_futex_value_locked API

The cmpxchg_futex_value_locked API was funny in that it returned either
the original, user-exposed futex value OR an error code such as -EFAULT.
This was confusing at best, and could be a source of livelocks in places
that retry the cmpxchg_futex_value_locked after trying to fix the issue
by running fault_in_user_writeable().

This change makes the cmpxchg_futex_value_locked API more similar to the
get_futex_value_locked one, returning an error code and updating the
original value through a reference argument.

Signed-off-by: Michel Lespinasse <walken@google.com>
Acked-by: Chris Metcalf <cmetcalf@tilera.com>  [tile]
Acked-by: Tony Luck <tony.luck@intel.com>  [ia64]
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Michal Simek <monstr@monstr.eu>  [microblaze]
Acked-by: David Howells <dhowells@redhat.com> [frv]
Cc: Darren Hart <darren@dvhart.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: "James E.J. Bottomley" <jejb@parisc-linux.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
LKML-Reference: <20110311024851.GC26122@google.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h
index 945de222ab91a3409836ce070a4bcd9f525061cb..c4e5c2850cce1e952549fe39cb3f3467ad347dc6 100644 (file)
--- a/arch/alpha/include/asm/futex.h
+++ b/arch/alpha/include/asm/futex.h
@@ -81,21 +81,22 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int prev, cmp;
+       int ret = 0, prev, cmp;
 
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
        __asm__ __volatile__ (
                __ASM_SMP_MB
-       "1:     ldl_l   %0,0(%2)\n"
-       "       cmpeq   %0,%3,%1\n"
-       "       beq     %1,3f\n"
-       "       mov     %4,%1\n"
-       "2:     stl_c   %1,0(%2)\n"
-       "       beq     %1,4f\n"
+       "1:     ldl_l   %1,0(%3)\n"
+       "       cmpeq   %1,%4,%2\n"
+       "       beq     %2,3f\n"
+       "       mov     %5,%2\n"
+       "2:     stl_c   %2,0(%3)\n"
+       "       beq     %2,4f\n"
        "3:     .subsection 2\n"
        "4:     br      1b\n"
        "       .previous\n"
@@ -105,11 +106,12 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
        "       .long   2b-.\n"
        "       lda     $31,3b-2b(%0)\n"
        "       .previous\n"
-       :       "=&r"(prev), "=&r"(cmp)
+       :       "+r"(ret), "=&r"(prev), "=&r"(cmp)
        :       "r"(uaddr), "r"((long)oldval), "r"(newval)
        :       "memory");
 
-       return prev;
+       *uval = prev;
+       return ret;
 }
 
 #endif /* __KERNEL__ */

diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h
index 7133a86208309a72d6bbfc9a42adf5219fe9136e..d20b78fce758fe1b406bb1528950893bc97b7221 100644 (file)
--- a/arch/arm/include/asm/futex.h
+++ b/arch/arm/include/asm/futex.h
@@ -88,9 +88,10 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int val;
+       int ret = 0, val;
 
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
@@ -99,24 +100,25 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
         * call sites. */
 
        __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
-       "1:     " T(ldr) "      %0, [%3]\n"
-       "       teq     %0, %1\n"
+       "1:     " T(ldr) "      %1, [%4]\n"
+       "       teq     %1, %2\n"
        "       it      eq      @ explicit IT needed for the 2b label\n"
-       "2:     " T(streq) "    %2, [%3]\n"
+       "2:     " T(streq) "    %3, [%4]\n"
        "3:\n"
        "       .pushsection __ex_table,\"a\"\n"
        "       .align  3\n"
        "       .long   1b, 4f, 2b, 4f\n"
        "       .popsection\n"
        "       .pushsection .fixup,\"ax\"\n"
-       "4:     mov     %0, %4\n"
+       "4:     mov     %0, %5\n"
        "       b       3b\n"
        "       .popsection"
-       : "=&r" (val)
+       : "+r" (ret), "=&r" (val)
        : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
        : "cc", "memory");
 
-       return val;
+       *uval = val;
+       return ret;
 }
 
 #endif /* !SMP */

diff --git a/arch/frv/include/asm/futex.h b/arch/frv/include/asm/futex.h
index 08b3d1da358398e111ccf01f50227ef8f60a7ffc..0548f8e4d11edfe0a5c13d27e41d2e077f18d041 100644 (file)
--- a/arch/frv/include/asm/futex.h
+++ b/arch/frv/include/asm/futex.h
@@ -10,7 +10,8 @@
 extern int futex_atomic_op_inuser(int encoded_op, int __user *uaddr);
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
        return -ENOSYS;
 }

diff --git a/arch/ia64/include/asm/futex.h b/arch/ia64/include/asm/futex.h
index c7f0f062239cd541112ecbe10cdd34dc54672eec..b0728404dad05c9349aea95d4e28b91b1b363a21 100644 (file)
--- a/arch/ia64/include/asm/futex.h
+++ b/arch/ia64/include/asm/futex.h
@@ -100,23 +100,26 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
        {
-               register unsigned long r8 __asm ("r8");
+               register unsigned long r8 __asm ("r8") = 0;
+               unsigned long prev;
                __asm__ __volatile__(
                        "       mf;;                                    \n"
                        "       mov ar.ccv=%3;;                         \n"
                        "[1:]   cmpxchg4.acq %0=[%1],%2,ar.ccv          \n"
                        "       .xdata4 \"__ex_table\", 1b-., 2f-.      \n"
                        "[2:]"
-                       : "=r" (r8)
+                       : "=r" (prev)
                        : "r" (uaddr), "r" (newval),
                          "rO" ((long) (unsigned) oldval)
                        : "memory");
+               *uval = prev;
                return r8;
        }
 }

diff --git a/arch/microblaze/include/asm/futex.h b/arch/microblaze/include/asm/futex.h
index ad3fd61b2fe7eff6689e72a9e6936093e226740d..fa019ed65dfb30054722e5c21a08b791909614a3 100644 (file)
--- a/arch/microblaze/include/asm/futex.h
+++ b/arch/microblaze/include/asm/futex.h
@@ -94,31 +94,33 @@ futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int prev, cmp;
+       int ret = 0, prev, cmp;
 
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       __asm__ __volatile__ ("1:       lwx     %0, %2, r0;             \
-                                       cmp     %1, %0, %3;             \
-                                       beqi    %1, 3f;                 \
-                               2:      swx     %4, %2, r0;             \
-                                       addic   %1, r0, 0;              \
-                                       bnei    %1, 1b;                 \
+       __asm__ __volatile__ ("1:       lwx     %1, %3, r0;             \
+                                       cmp     %2, %1, %4;             \
+                                       beqi    %2, 3f;                 \
+                               2:      swx     %5, %3, r0;             \
+                                       addic   %2, r0, 0;              \
+                                       bnei    %2, 1b;                 \
                                3:                                      \
                                .section .fixup,\"ax\";                 \
                                4:      brid    3b;                     \
-                                       addik   %0, r0, %5;             \
+                                       addik   %0, r0, %6;             \
                                .previous;                              \
                                .section __ex_table,\"a\";              \
                                .word   1b,4b,2b,4b;                    \
                                .previous;"                             \
-               : "=&r" (prev), "=&r"(cmp)                              \
+               : "+r" (ret), "=&r" (prev), "=&r"(cmp)  \
                : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT));
 
-       return prev;
+       *uval = prev;
+       return ret;
 }
 
 #endif /* __KERNEL__ */

diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h
index b9cce90346cfc334819f44e95c4308f96b8f9533..692a24bd83b7aa9b14a71c512edc76b549911f4f 100644 (file)
--- a/arch/mips/include/asm/futex.h
+++ b/arch/mips/include/asm/futex.h
@@ -132,9 +132,10 @@ futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int retval;
+       int ret = 0, val;
 
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
@@ -145,25 +146,25 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
                "       .set    push                                    \n"
                "       .set    noat                                    \n"
                "       .set    mips3                                   \n"
-               "1:     ll      %0, %2                                  \n"
-               "       bne     %0, %z3, 3f                             \n"
+               "1:     ll      %1, %3                                  \n"
+               "       bne     %1, %z4, 3f                             \n"
                "       .set    mips0                                   \n"
-               "       move    $1, %z4                                 \n"
+               "       move    $1, %z5                                 \n"
                "       .set    mips3                                   \n"
-               "2:     sc      $1, %1                                  \n"
+               "2:     sc      $1, %2                                  \n"
                "       beqzl   $1, 1b                                  \n"
                __WEAK_LLSC_MB
                "3:                                                     \n"
                "       .set    pop                                     \n"
                "       .section .fixup,\"ax\"                          \n"
-               "4:     li      %0, %5                                  \n"
+               "4:     li      %0, %6                                  \n"
                "       j       3b                                      \n"
                "       .previous                                       \n"
                "       .section __ex_table,\"a\"                       \n"
                "       "__UA_ADDR "\t1b, 4b                            \n"
                "       "__UA_ADDR "\t2b, 4b                            \n"
                "       .previous                                       \n"
-               : "=&r" (retval), "=R" (*uaddr)
+               : "+r" (ret), "=&r" (val), "=R" (*uaddr)
                : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT)
                : "memory");
        } else if (cpu_has_llsc) {
@@ -172,31 +173,32 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
                "       .set    push                                    \n"
                "       .set    noat                                    \n"
                "       .set    mips3                                   \n"
-               "1:     ll      %0, %2                                  \n"
-               "       bne     %0, %z3, 3f                             \n"
+               "1:     ll      %1, %3                                  \n"
+               "       bne     %1, %z4, 3f                             \n"
                "       .set    mips0                                   \n"
-               "       move    $1, %z4                                 \n"
+               "       move    $1, %z5                                 \n"
                "       .set    mips3                                   \n"
-               "2:     sc      $1, %1                                  \n"
+               "2:     sc      $1, %2                                  \n"
                "       beqz    $1, 1b                                  \n"
                __WEAK_LLSC_MB
                "3:                                                     \n"
                "       .set    pop                                     \n"
                "       .section .fixup,\"ax\"                          \n"
-               "4:     li      %0, %5                                  \n"
+               "4:     li      %0, %6                                  \n"
                "       j       3b                                      \n"
                "       .previous                                       \n"
                "       .section __ex_table,\"a\"                       \n"
                "       "__UA_ADDR "\t1b, 4b                            \n"
                "       "__UA_ADDR "\t2b, 4b                            \n"
                "       .previous                                       \n"
-               : "=&r" (retval), "=R" (*uaddr)
+               : "+r" (ret), "=&r" (val), "=R" (*uaddr)
                : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT)
                : "memory");
        } else
                return -ENOSYS;
 
-       return retval;
+       *uval = val;
+       return ret;
 }
 
 #endif

diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h
index 0c705c3a55efc01305dbea4e14eecf63c0f3f05e..4c6d8672325b875dffad38efb917b9fad06ba5a5 100644 (file)
--- a/arch/parisc/include/asm/futex.h
+++ b/arch/parisc/include/asm/futex.h
@@ -51,10 +51,10 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 
 /* Non-atomic version */
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int err = 0;
-       int uval;
+       int val;
 
        /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
         * our gateway page, and causes no end of trouble...
@@ -65,12 +65,12 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       err = get_user(uval, uaddr);
-       if (err) return -EFAULT;
-       if (uval == oldval)
-               err = put_user(newval, uaddr);
-       if (err) return -EFAULT;
-       return uval;
+       if (get_user(val, uaddr))
+               return -EFAULT;
+       if (val == oldval && put_user(newval, uaddr))
+               return -EFAULT;
+       *uval = val;
+       return 0;
 }
 
 #endif /*__KERNEL__*/

diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h
index 7c589ef81fb0eec42eee5f17cfbfc25563bb66fa..631e8da60064801c3c7837897dcbad9c07230ff5 100644 (file)
--- a/arch/powerpc/include/asm/futex.h
+++ b/arch/powerpc/include/asm/futex.h
@@ -82,35 +82,37 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
-       int prev;
+       int ret = 0, prev;
 
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
         __asm__ __volatile__ (
         PPC_RELEASE_BARRIER
-"1:     lwarx   %0,0,%2         # futex_atomic_cmpxchg_inatomic\n\
-        cmpw    0,%0,%3\n\
+"1:     lwarx   %1,0,%3         # futex_atomic_cmpxchg_inatomic\n\
+        cmpw    0,%1,%4\n\
         bne-    3f\n"
-        PPC405_ERR77(0,%2)
-"2:     stwcx.  %4,0,%2\n\
+        PPC405_ERR77(0,%3)
+"2:     stwcx.  %5,0,%3\n\
         bne-    1b\n"
         PPC_ACQUIRE_BARRIER
 "3:    .section .fixup,\"ax\"\n\
-4:     li      %0,%5\n\
+4:     li      %0,%6\n\
        b       3b\n\
        .previous\n\
        .section __ex_table,\"a\"\n\
        .align 3\n\
        " PPC_LONG "1b,4b,2b,4b\n\
        .previous" \
-        : "=&r" (prev), "+m" (*uaddr)
+        : "+r" (ret), "=&r" (prev), "+m" (*uaddr)
         : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)
         : "cc", "memory");
 
-        return prev;
+       *uval = prev;
+        return ret;
 }
 
 #endif /* __KERNEL__ */

diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h
index 5c5d02de49e9ae53fe5728fef181f1657fd4c3dc..27ac515ef59c9981f2cd83cccf1e5fe8d6095d71 100644 (file)
--- a/arch/s390/include/asm/futex.h
+++ b/arch/s390/include/asm/futex.h
@@ -39,13 +39,13 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
        return ret;
 }
 
-static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr,
+static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
                                                int oldval, int newval)
 {
        if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       return uaccess.futex_atomic_cmpxchg(uaddr, oldval, newval);
+       return uaccess.futex_atomic_cmpxchg(uval, uaddr, oldval, newval);
 }
 
 #endif /* __KERNEL__ */

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index d6b1ed0ec52b3b8bb5a27537a2fa378cfddce34b..549adf6a9b8b3c198ec0b52688f9268d3256530f 100644 (file)
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -84,7 +84,7 @@ struct uaccess_ops {
        size_t (*strnlen_user)(size_t, const char __user *);
        size_t (*strncpy_from_user)(size_t, const char __user *, char *);
        int (*futex_atomic_op)(int op, int __user *, int oparg, int *old);
-       int (*futex_atomic_cmpxchg)(int __user *, int old, int new);
+       int (*futex_atomic_cmpxchg)(int *, int __user *, int old, int new);
 };
 
 extern struct uaccess_ops uaccess;

diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h
index 126011df14f1e7ad1a582e90de001177330c66d8..89a80674e44b064da9a9a1272b9d34aed13a79dd 100644 (file)
--- a/arch/s390/lib/uaccess.h
+++ b/arch/s390/lib/uaccess.h
@@ -12,12 +12,12 @@ extern size_t copy_from_user_std(size_t, const void __user *, void *);
 extern size_t copy_to_user_std(size_t, void __user *, const void *);
 extern size_t strnlen_user_std(size_t, const char __user *);
 extern size_t strncpy_from_user_std(size_t, const char __user *, char *);
-extern int futex_atomic_cmpxchg_std(int __user *, int, int);
+extern int futex_atomic_cmpxchg_std(int *, int __user *, int, int);
 extern int futex_atomic_op_std(int, int __user *, int, int *);
 
 extern size_t copy_from_user_pt(size_t, const void __user *, void *);
 extern size_t copy_to_user_pt(size_t, void __user *, const void *);
 extern int futex_atomic_op_pt(int, int __user *, int, int *);
-extern int futex_atomic_cmpxchg_pt(int __user *, int, int);
+extern int futex_atomic_cmpxchg_pt(int *, int __user *, int, int);
 
 #endif /* __ARCH_S390_LIB_UACCESS_H */

diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 404f2de296dcad3c6017488363dc183986ad0bf1..b3cebcd52f5c11a2d0ac50c4980c131c4a94aee7 100644 (file)
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -354,26 +354,29 @@ int futex_atomic_op_pt(int op, int __user *uaddr, int oparg, int *old)
        return ret;
 }
 
-static int __futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval)
+static int __futex_atomic_cmpxchg_pt(int *uval, int __user *uaddr,
+                                    int oldval, int newval)
 {
        int ret;
 
        asm volatile("0: cs   %1,%4,0(%5)\n"
-                    "1: lr   %0,%1\n"
+                    "1: la   %0,0\n"
                     "2:\n"
                     EX_TABLE(0b,2b) EX_TABLE(1b,2b)
                     : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
                     : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
                     : "cc", "memory" );
+       *uval = oldval;
        return ret;
 }
 
-int futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval)
+int futex_atomic_cmpxchg_pt(int *uval, int __user *uaddr,
+                           int oldval, int newval)
 {
        int ret;
 
        if (segment_eq(get_fs(), KERNEL_DS))
-               return __futex_atomic_cmpxchg_pt(uaddr, oldval, newval);
+               return __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
        spin_lock(&current->mm->page_table_lock);
        uaddr = (int __user *) __dat_user_addr((unsigned long) uaddr);
        if (!uaddr) {
@@ -382,7 +385,7 @@ int futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval)
        }
        get_page(virt_to_page(uaddr));
        spin_unlock(&current->mm->page_table_lock);
-       ret = __futex_atomic_cmpxchg_pt(uaddr, oldval, newval);
+       ret = __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
        put_page(virt_to_page(uaddr));
        return ret;
 }

diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c
index a6c4f7ed24a493d1e0ea5fde43b19dc5b0fe37f2..1d6643c0b95f9ae5e26a7f5af3c03c7fd064f248 100644 (file)
--- a/arch/s390/lib/uaccess_std.c
+++ b/arch/s390/lib/uaccess_std.c
@@ -287,19 +287,21 @@ int futex_atomic_op_std(int op, int __user *uaddr, int oparg, int *old)
        return ret;
 }
 
-int futex_atomic_cmpxchg_std(int __user *uaddr, int oldval, int newval)
+int futex_atomic_cmpxchg_std(int *uval, int __user *uaddr,
+                            int oldval, int newval)
 {
        int ret;
 
        asm volatile(
                "   sacf 256\n"
                "0: cs   %1,%4,0(%5)\n"
-               "1: lr   %0,%1\n"
+               "1: la   %0,0\n"
                "2: sacf 0\n"
                EX_TABLE(0b,2b) EX_TABLE(1b,2b)
                : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
                : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
                : "cc", "memory" );
+       *uval = oldval;
        return ret;
 }
 

diff --git a/arch/sh/include/asm/futex-irq.h b/arch/sh/include/asm/futex-irq.h
index a9f16a7f9aeaf5bf1a1482c8e38355b1c35c6f2d..7b701cbd1e84afd8c06e394500e6343f496ed45e 100644 (file)
--- a/arch/sh/include/asm/futex-irq.h
+++ b/arch/sh/include/asm/futex-irq.h
@@ -88,7 +88,8 @@ static inline int atomic_futex_op_xchg_xor(int oparg, int __user *uaddr,
        return ret;
 }
 
-static inline int atomic_futex_op_cmpxchg_inatomic(int __user *uaddr,
+static inline int atomic_futex_op_cmpxchg_inatomic(int *uval,
+                                                  int __user *uaddr,
                                                   int oldval, int newval)
 {
        unsigned long flags;
@@ -102,10 +103,8 @@ static inline int atomic_futex_op_cmpxchg_inatomic(int __user *uaddr,
 
        local_irq_restore(flags);
 
-       if (ret)
-               return ret;
-
-       return prev;
+       *uval = prev;
+       return ret;
 }
 
 #endif /* __ASM_SH_FUTEX_IRQ_H */

diff --git a/arch/sh/include/asm/futex.h b/arch/sh/include/asm/futex.h
index 68256ec5fa35b3e2c46ead9e2566810583351dcd..a8a5125dc9b43dc973167af680d63ff4edc18676 100644 (file)
--- a/arch/sh/include/asm/futex.h
+++ b/arch/sh/include/asm/futex.h
@@ -65,12 +65,13 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       return atomic_futex_op_cmpxchg_inatomic(uaddr, oldval, newval);
+       return atomic_futex_op_cmpxchg_inatomic(uval, uaddr, oldval, newval);
 }
 
 #endif /* __KERNEL__ */

diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h
index 47f95839dc6956e190cd1175cb50b74491dcab00..e0862200d6a1460dc3bef96f745c104e49b0e536 100644 (file)
--- a/arch/sparc/include/asm/futex_64.h
+++ b/arch/sparc/include/asm/futex_64.h
@@ -85,26 +85,30 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
+       int ret = 0;
+
        __asm__ __volatile__(
-       "\n1:   casa    [%3] %%asi, %2, %0\n"
+       "\n1:   casa    [%4] %%asi, %3, %1\n"
        "2:\n"
        "       .section .fixup,#alloc,#execinstr\n"
        "       .align  4\n"
        "3:     sethi   %%hi(2b), %0\n"
        "       jmpl    %0 + %%lo(2b), %%g0\n"
-       "        mov    %4, %0\n"
+       "       mov     %5, %0\n"
        "       .previous\n"
        "       .section __ex_table,\"a\"\n"
        "       .align  4\n"
        "       .word   1b, 3b\n"
        "       .previous\n"
-       : "=r" (newval)
-       : "0" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT)
+       : "+r" (ret), "=r" (newval)
+       : "1" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT)
        : "memory");
 
-       return newval;
+       *uval = newval;
+       return ret;
 }
 
 #endif /* !(_SPARC64_FUTEX_H) */

diff --git a/arch/tile/include/asm/futex.h b/arch/tile/include/asm/futex.h
index fe0d10dcae57a140d6fa083b8a815b65d89d965b..664b20aa2584fa0a57b675291c0caefcb46cdf37 100644 (file)
--- a/arch/tile/include/asm/futex.h
+++ b/arch/tile/include/asm/futex.h
@@ -119,8 +119,8 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
        return ret;
 }
 
-static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
-                                               int newval)
+static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                                               int oldval, int newval)
 {
        struct __get_user asm_ret;
 
@@ -128,7 +128,8 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
                return -EFAULT;
 
        asm_ret = futex_cmpxchg(uaddr, oldval, newval);
-       return asm_ret.err ? asm_ret.err : asm_ret.val;
+       *uval = asm_ret.val;
+       return asm_ret.err;
 }
 
 #ifndef __tilegx__

diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h
index 1f11ce44e956dc41d3e33812821aa8249455a269..884c0b5676f4b549fded8291b93b2f100464f081 100644 (file)
--- a/arch/x86/include/asm/futex.h
+++ b/arch/x86/include/asm/futex.h
@@ -109,9 +109,10 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
        return ret;
 }
 
-static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
-                                               int newval)
+static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                                               int oldval, int newval)
 {
+       int ret = 0;
 
 #if defined(CONFIG_X86_32) && !defined(CONFIG_X86_BSWAP)
        /* Real i386 machines have no cmpxchg instruction */
@@ -122,18 +123,19 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
        if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                return -EFAULT;
 
-       asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %3, %1\n"
+       asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"
                     "2:\t.section .fixup, \"ax\"\n"
-                    "3:\tmov     %2, %0\n"
+                    "3:\tmov     %3, %0\n"
                     "\tjmp     2b\n"
                     "\t.previous\n"
                     _ASM_EXTABLE(1b, 3b)
-                    : "=a" (oldval), "+m" (*uaddr)
-                    : "i" (-EFAULT), "r" (newval), "0" (oldval)
+                    : "+r" (ret), "=a" (oldval), "+m" (*uaddr)
+                    : "i" (-EFAULT), "r" (newval), "1" (oldval)
                     : "memory"
        );
 
-       return oldval;
+       *uval = oldval;
+       return ret;
 }
 
 #endif

diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index 3c2344f48136d98351f92b6e9d678e29c052658c..132bf5227b448739f734b7ae69619c60e42b5b07 100644 (file)
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -48,7 +48,8 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
+futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
+                             int oldval, int newval)
 {
        return -ENOSYS;
 }

diff --git a/kernel/futex.c b/kernel/futex.c
index 773815465bace17d40e9b014a1dd352809697d78..237f14bfc02226c26e4eca7b38289f48d7d39416 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -381,15 +381,16 @@ static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,
        return NULL;
 }
 
-static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval)
+static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr,
+                                     u32 uval, u32 newval)
 {
-       u32 curval;
+       int ret;
 
        pagefault_disable();
-       curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
+       ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
        pagefault_enable();
 
-       return curval;
+       return ret;
 }
 
 static int get_futex_value_locked(u32 *dest, u32 __user *from)
@@ -688,9 +689,7 @@ retry:
        if (set_waiters)
                newval |= FUTEX_WAITERS;
 
-       curval = cmpxchg_futex_value_locked(uaddr, 0, newval);
-
-       if (unlikely(curval == -EFAULT))
+       if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, 0, newval)))
                return -EFAULT;
 
        /*
@@ -728,9 +727,7 @@ retry:
                lock_taken = 1;
        }
 
-       curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
-
-       if (unlikely(curval == -EFAULT))
+       if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
                return -EFAULT;
        if (unlikely(curval != uval))
                goto retry;
@@ -843,9 +840,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
 
                newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
 
-               curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
-
-               if (curval == -EFAULT)
+               if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
                        ret = -EFAULT;
                else if (curval != uval)
                        ret = -EINVAL;
@@ -880,10 +875,8 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
         * There is no waiter, so we unlock the futex. The owner died
         * bit has not to be preserved here. We are the owner:
         */
-       oldval = cmpxchg_futex_value_locked(uaddr, uval, 0);
-
-       if (oldval == -EFAULT)
-               return oldval;
+       if (cmpxchg_futex_value_locked(&oldval, uaddr, uval, 0))
+               return -EFAULT;
        if (oldval != uval)
                return -EAGAIN;
 
@@ -1578,9 +1571,7 @@ retry:
        while (1) {
                newval = (uval & FUTEX_OWNER_DIED) | newtid;
 
-               curval = cmpxchg_futex_value_locked(uaddr, uval, newval);
-
-               if (curval == -EFAULT)
+               if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
                        goto handle_fault;
                if (curval == uval)
                        break;
@@ -2073,11 +2064,8 @@ retry:
         * again. If it succeeds then we can return without waking
         * anyone else up:
         */
-       if (!(uval & FUTEX_OWNER_DIED))
-               uval = cmpxchg_futex_value_locked(uaddr, vpid, 0);
-
-
-       if (unlikely(uval == -EFAULT))
+       if (!(uval & FUTEX_OWNER_DIED) &&
+           cmpxchg_futex_value_locked(&uval, uaddr, vpid, 0))
                goto pi_faulted;
        /*
         * Rare case: we managed to release the lock atomically,
@@ -2464,9 +2452,7 @@ retry:
                 * userspace.
                 */
                mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
-               nval = futex_atomic_cmpxchg_inatomic(uaddr, uval, mval);
-
-               if (nval == -EFAULT)
+               if (futex_atomic_cmpxchg_inatomic(&nval, uaddr, uval, mval))
                        return -1;
 
                if (nval != uval)
@@ -2679,8 +2665,7 @@ static int __init futex_init(void)
         * implementation, the non-functional ones will return
         * -ENOSYS.
         */
-       curval = cmpxchg_futex_value_locked(NULL, 0, 0);
-       if (curval == -EFAULT)
+       if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
                futex_cmpxchg_enabled = 1;
 
        for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {