1 #include <linux/compiler.h>
2 #include <linux/module.h>
3 #include <linux/types.h>
5 #include <asm/processor.h>
6 #include <asm/cmpxchg.h>
7 #include <asm/atomic.h>
9 static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new)
15 LOCK_PREFIX "cmpxchg8b %1\n"
16 : "+A" (old), "+m" (*ptr)
17 : "b" (low), "c" (high)
22 u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val)
24 return cmpxchg8b(&ptr->counter, old_val, new_val);
26 EXPORT_SYMBOL(atomic64_cmpxchg);
29 * atomic64_xchg - xchg atomic64 variable
30 * @ptr: pointer to type atomic64_t
31 * @new_val: value to assign
33 * Atomically xchgs the value of @ptr to @new_val and returns
36 u64 atomic64_xchg(atomic64_t *ptr, u64 new_val)
39 * Try first with a (possibly incorrect) assumption about
40 * what we have there. We'll do two loops most likely,
41 * but we'll get an ownership MESI transaction straight away
42 * instead of a read transaction followed by a
43 * flush-for-ownership transaction:
45 u64 old_val, real_val = 0;
50 real_val = atomic64_cmpxchg(ptr, old_val, new_val);
52 } while (real_val != old_val);
56 EXPORT_SYMBOL(atomic64_xchg);
59 * atomic64_set - set atomic64 variable
60 * @ptr: pointer to type atomic64_t
61 * @new_val: value to assign
63 * Atomically sets the value of @ptr to @new_val.
65 void atomic64_set(atomic64_t *ptr, u64 new_val)
67 atomic64_xchg(ptr, new_val);
69 EXPORT_SYMBOL(atomic64_set);
72 EXPORT_SYMBOL(atomic64_read);
73 * atomic64_add_return - add and return
74 * @delta: integer value to add
75 * @ptr: pointer to type atomic64_t
77 * Atomically adds @delta to @ptr and returns @delta + *@ptr
79 noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr)
82 * Try first with a (possibly incorrect) assumption about
83 * what we have there. We'll do two loops most likely,
84 * but we'll get an ownership MESI transaction straight away
85 * instead of a read transaction followed by a
86 * flush-for-ownership transaction:
88 u64 old_val, new_val, real_val = 0;
92 new_val = old_val + delta;
94 real_val = atomic64_cmpxchg(ptr, old_val, new_val);
96 } while (real_val != old_val);
100 EXPORT_SYMBOL(atomic64_add_return);
102 u64 atomic64_sub_return(u64 delta, atomic64_t *ptr)
104 return atomic64_add_return(-delta, ptr);
106 EXPORT_SYMBOL(atomic64_sub_return);
108 u64 atomic64_inc_return(atomic64_t *ptr)
110 return atomic64_add_return(1, ptr);
112 EXPORT_SYMBOL(atomic64_inc_return);
114 u64 atomic64_dec_return(atomic64_t *ptr)
116 return atomic64_sub_return(1, ptr);
118 EXPORT_SYMBOL(atomic64_dec_return);
121 * atomic64_add - add integer to atomic64 variable
122 * @delta: integer value to add
123 * @ptr: pointer to type atomic64_t
125 * Atomically adds @delta to @ptr.
127 void atomic64_add(u64 delta, atomic64_t *ptr)
129 atomic64_add_return(delta, ptr);
131 EXPORT_SYMBOL(atomic64_add);
134 * atomic64_sub - subtract the atomic64 variable
135 * @delta: integer value to subtract
136 * @ptr: pointer to type atomic64_t
138 * Atomically subtracts @delta from @ptr.
140 void atomic64_sub(u64 delta, atomic64_t *ptr)
142 atomic64_add(-delta, ptr);
144 EXPORT_SYMBOL(atomic64_sub);
147 * atomic64_sub_and_test - subtract value from variable and test result
148 * @delta: integer value to subtract
149 * @ptr: pointer to type atomic64_t
151 * Atomically subtracts @delta from @ptr and returns
152 * true if the result is zero, or false for all
155 int atomic64_sub_and_test(u64 delta, atomic64_t *ptr)
157 u64 new_val = atomic64_sub_return(delta, ptr);
161 EXPORT_SYMBOL(atomic64_sub_and_test);
164 * atomic64_inc - increment atomic64 variable
165 * @ptr: pointer to type atomic64_t
167 * Atomically increments @ptr by 1.
169 void atomic64_inc(atomic64_t *ptr)
171 atomic64_add(1, ptr);
173 EXPORT_SYMBOL(atomic64_inc);
176 * atomic64_dec - decrement atomic64 variable
177 * @ptr: pointer to type atomic64_t
179 * Atomically decrements @ptr by 1.
181 void atomic64_dec(atomic64_t *ptr)
183 atomic64_sub(1, ptr);
185 EXPORT_SYMBOL(atomic64_dec);
188 * atomic64_dec_and_test - decrement and test
189 * @ptr: pointer to type atomic64_t
191 * Atomically decrements @ptr by 1 and
192 * returns true if the result is 0, or false for all other
195 int atomic64_dec_and_test(atomic64_t *ptr)
197 return atomic64_sub_and_test(1, ptr);
199 EXPORT_SYMBOL(atomic64_dec_and_test);
202 * atomic64_inc_and_test - increment and test
203 * @ptr: pointer to type atomic64_t
205 * Atomically increments @ptr by 1
206 * and returns true if the result is zero, or false for all
209 int atomic64_inc_and_test(atomic64_t *ptr)
211 return atomic64_sub_and_test(-1, ptr);
213 EXPORT_SYMBOL(atomic64_inc_and_test);
216 * atomic64_add_negative - add and test if negative
217 * @delta: integer value to add
218 * @ptr: pointer to type atomic64_t
220 * Atomically adds @delta to @ptr and returns true
221 * if the result is negative, or false when
222 * result is greater than or equal to zero.
224 int atomic64_add_negative(u64 delta, atomic64_t *ptr)
226 s64 new_val = atomic64_add_return(delta, ptr);
230 EXPORT_SYMBOL(atomic64_add_negative);