arch/arm64/include/asm/atomic_ll_sc.h

   1 /*
   2  * Based on arch/arm/include/asm/atomic.h
   3  *
   4  * Copyright (C) 1996 Russell King.
   5  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   6  * Copyright (C) 2012 ARM Ltd.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #ifndef __ASM_ATOMIC_LL_SC_H
  22 #define __ASM_ATOMIC_LL_SC_H
  23
  24 #ifndef __ARM64_IN_ATOMIC_IMPL
  25 #error "please don't include this file directly"
  26 #endif
  27
  28 /*
  29  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
  30  * store exclusive to ensure that these are atomic.  We may loop
  31  * to ensure that the update happens.
  32  *
  33  * NOTE: these functions do *not* follow the PCS and must explicitly
  34  * save any clobbered registers other than x0 (regardless of return
  35  * value).  This is achieved through -fcall-saved-* compiler flags for
  36  * this file, which unfortunately don't work on a per-function basis
  37  * (the optimize attribute silently ignores these options).
  38  */
  39
  40 #define ATOMIC_OP(op, asm_op)                                           \
  41 __LL_SC_INLINE void                                                     \
  42 __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v))                         \
  43 {                                                                       \
  44         unsigned long tmp;                                              \
  45         int result;                                                     \
  46                                                                         \
  47         asm volatile("// atomic_" #op "\n"                              \
  48 "1:     ldxr    %w0, %2\n"                                              \
  49 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  50 "       stxr    %w1, %w0, %2\n"                                         \
  51 "       cbnz    %w1, 1b"                                                \
  52         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  53         : "Ir" (i));                                                    \
  54 }                                                                       \
  55 __LL_SC_EXPORT(atomic_##op);
  56
  57 #define ATOMIC_OP_RETURN(op, asm_op)                                    \
  58 __LL_SC_INLINE int                                                      \
  59 __LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v))                \
  60 {                                                                       \
  61         unsigned long tmp;                                              \
  62         int result;                                                     \
  63                                                                         \
  64         asm volatile("// atomic_" #op "_return\n"                       \
  65 "1:     ldxr    %w0, %2\n"                                              \
  66 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  67 "       stlxr   %w1, %w0, %2\n"                                         \
  68 "       cbnz    %w1, 1b"                                                \
  69         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  70         : "Ir" (i)                                                      \
  71         : "memory");                                                    \
  72                                                                         \
  73         smp_mb();                                                       \
  74         return result;                                                  \
  75 }                                                                       \
  76 __LL_SC_EXPORT(atomic_##op##_return);
  77
  78 #define ATOMIC_OPS(op, asm_op)                                          \
  79         ATOMIC_OP(op, asm_op)                                           \
  80         ATOMIC_OP_RETURN(op, asm_op)
  81
  82 ATOMIC_OPS(add, add)
  83 ATOMIC_OPS(sub, sub)
  84
  85 ATOMIC_OP(and, and)
  86 ATOMIC_OP(andnot, bic)
  87 ATOMIC_OP(or, orr)
  88 ATOMIC_OP(xor, eor)
  89
  90 #undef ATOMIC_OPS
  91 #undef ATOMIC_OP_RETURN
  92 #undef ATOMIC_OP
  93
  94 __LL_SC_INLINE int
  95 __LL_SC_PREFIX(atomic_cmpxchg(atomic_t *ptr, int old, int new))
  96 {
  97         unsigned long tmp;
  98         int oldval;
  99
 100         smp_mb();
 101
 102         asm volatile("// atomic_cmpxchg\n"
 103 "1:     ldxr    %w1, %2\n"
 104 "       eor     %w0, %w1, %w3\n"
 105 "       cbnz    %w0, 2f\n"
 106 "       stxr    %w0, %w4, %2\n"
 107 "       cbnz    %w0, 1b\n"
 108 "2:"
 109         : "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
 110         : "Lr" (old), "r" (new));
 111
 112         smp_mb();
 113         return oldval;
 114 }
 115 __LL_SC_EXPORT(atomic_cmpxchg);
 116
 117 #define ATOMIC64_OP(op, asm_op)                                         \
 118 __LL_SC_INLINE void                                                     \
 119 __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v))                    \
 120 {                                                                       \
 121         long result;                                                    \
 122         unsigned long tmp;                                              \
 123                                                                         \
 124         asm volatile("// atomic64_" #op "\n"                            \
 125 "1:     ldxr    %0, %2\n"                                               \
 126 "       " #asm_op "     %0, %0, %3\n"                                   \
 127 "       stxr    %w1, %0, %2\n"                                          \
 128 "       cbnz    %w1, 1b"                                                \
 129         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 130         : "Ir" (i));                                                    \
 131 }                                                                       \
 132 __LL_SC_EXPORT(atomic64_##op);
 133
 134 #define ATOMIC64_OP_RETURN(op, asm_op)                                  \
 135 __LL_SC_INLINE long                                                     \
 136 __LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v))           \
 137 {                                                                       \
 138         long result;                                                    \
 139         unsigned long tmp;                                              \
 140                                                                         \
 141         asm volatile("// atomic64_" #op "_return\n"                     \
 142 "1:     ldxr    %0, %2\n"                                               \
 143 "       " #asm_op "     %0, %0, %3\n"                                   \
 144 "       stlxr   %w1, %0, %2\n"                                          \
 145 "       cbnz    %w1, 1b"                                                \
 146         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 147         : "Ir" (i)                                                      \
 148         : "memory");                                                    \
 149                                                                         \
 150         smp_mb();                                                       \
 151         return result;                                                  \
 152 }                                                                       \
 153 __LL_SC_EXPORT(atomic64_##op##_return);
 154
 155 #define ATOMIC64_OPS(op, asm_op)                                        \
 156         ATOMIC64_OP(op, asm_op)                                         \
 157         ATOMIC64_OP_RETURN(op, asm_op)
 158
 159 ATOMIC64_OPS(add, add)
 160 ATOMIC64_OPS(sub, sub)
 161
 162 ATOMIC64_OP(and, and)
 163 ATOMIC64_OP(andnot, bic)
 164 ATOMIC64_OP(or, orr)
 165 ATOMIC64_OP(xor, eor)
 166
 167 #undef ATOMIC64_OPS
 168 #undef ATOMIC64_OP_RETURN
 169 #undef ATOMIC64_OP
 170
 171 __LL_SC_INLINE long
 172 __LL_SC_PREFIX(atomic64_cmpxchg(atomic64_t *ptr, long old, long new))
 173 {
 174         long oldval;
 175         unsigned long res;
 176
 177         smp_mb();
 178
 179         asm volatile("// atomic64_cmpxchg\n"
 180 "1:     ldxr    %1, %2\n"
 181 "       eor     %0, %1, %3\n"
 182 "       cbnz    %w0, 2f\n"
 183 "       stxr    %w0, %4, %2\n"
 184 "       cbnz    %w0, 1b\n"
 185 "2:"
 186         : "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
 187         : "Lr" (old), "r" (new));
 188
 189         smp_mb();
 190         return oldval;
 191 }
 192 __LL_SC_EXPORT(atomic64_cmpxchg);
 193
 194 __LL_SC_INLINE long
 195 __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
 196 {
 197         long result;
 198         unsigned long tmp;
 199
 200         asm volatile("// atomic64_dec_if_positive\n"
 201 "1:     ldxr    %0, %2\n"
 202 "       subs    %0, %0, #1\n"
 203 "       b.mi    2f\n"
 204 "       stlxr   %w1, %0, %2\n"
 205 "       cbnz    %w1, 1b\n"
 206 "       dmb     ish\n"
 207 "2:"
 208         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
 209         :
 210         : "cc", "memory");
 211
 212         return result;
 213 }
 214 __LL_SC_EXPORT(atomic64_dec_if_positive);
 215
 216 #define __CMPXCHG_CASE(w, sz, name, mb, cl)                             \
 217 __LL_SC_INLINE unsigned long                                            \
 218 __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr,                \
 219                                      unsigned long old,                 \
 220                                      unsigned long new))                \
 221 {                                                                       \
 222         unsigned long tmp, oldval;                                      \
 223                                                                         \
 224         asm volatile(                                                   \
 225         "       " #mb "\n"                                              \
 226         "1:     ldxr" #sz "\t%" #w "[oldval], %[v]\n"                   \
 227         "       eor     %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"  \
 228         "       cbnz    %" #w "[tmp], 2f\n"                             \
 229         "       stxr" #sz "\t%w[tmp], %" #w "[new], %[v]\n"             \
 230         "       cbnz    %w[tmp], 1b\n"                                  \
 231         "       " #mb "\n"                                              \
 232         "       mov     %" #w "[oldval], %" #w "[old]\n"                \
 233         "2:"                                                            \
 234         : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                   \
 235           [v] "+Q" (*(unsigned long *)ptr)                              \
 236         : [old] "Lr" (old), [new] "r" (new)                             \
 237         : cl);                                                          \
 238                                                                         \
 239         return oldval;                                                  \
 240 }                                                                       \
 241 __LL_SC_EXPORT(__cmpxchg_case_##name);
 242
 243 __CMPXCHG_CASE(w, b,    1,        ,         )
 244 __CMPXCHG_CASE(w, h,    2,        ,         )
 245 __CMPXCHG_CASE(w,  ,    4,        ,         )
 246 __CMPXCHG_CASE( ,  ,    8,        ,         )
 247 __CMPXCHG_CASE(w, b, mb_1, dmb ish, "memory")
 248 __CMPXCHG_CASE(w, h, mb_2, dmb ish, "memory")
 249 __CMPXCHG_CASE(w,  , mb_4, dmb ish, "memory")
 250 __CMPXCHG_CASE( ,  , mb_8, dmb ish, "memory")
 251
 252 #undef __CMPXCHG_CASE
 253
 254 #define __CMPXCHG_DBL(name, mb, cl)                                     \
 255 __LL_SC_INLINE int                                                      \
 256 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,               \
 257                                       unsigned long old2,               \
 258                                       unsigned long new1,               \
 259                                       unsigned long new2,               \
 260                                       volatile void *ptr))              \
 261 {                                                                       \
 262         unsigned long tmp, ret;                                         \
 263                                                                         \
 264         asm volatile("// __cmpxchg_double" #name "\n"                   \
 265         "       " #mb "\n"                                              \
 266         "1:     ldxp    %0, %1, %2\n"                                   \
 267         "       eor     %0, %0, %3\n"                                   \
 268         "       eor     %1, %1, %4\n"                                   \
 269         "       orr     %1, %0, %1\n"                                   \
 270         "       cbnz    %1, 2f\n"                                       \
 271         "       stxp    %w0, %5, %6, %2\n"                              \
 272         "       cbnz    %w0, 1b\n"                                      \
 273         "       " #mb "\n"                                              \
 274         "2:"                                                            \
 275         : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)        \
 276         : "r" (old1), "r" (old2), "r" (new1), "r" (new2)                \
 277         : cl);                                                          \
 278                                                                         \
 279         return ret;                                                     \
 280 }                                                                       \
 281 __LL_SC_EXPORT(__cmpxchg_double##name);
 282
 283 __CMPXCHG_DBL(   ,        ,         )
 284 __CMPXCHG_DBL(_mb, dmb ish, "memory")
 285
 286 #undef __CMPXCHG_DBL
 287
 288 #endif  /* __ASM_ATOMIC_LL_SC_H */