2 * Copyright 2007-2008 Analog Devices Inc.
3 * Philippe Gerum <rpm@xenomai.org>
5 * Licensed under the GPL-2 or later.
8 #include <linux/linkage.h>
9 #include <asm/blackfin.h>
10 #include <asm/cache.h>
11 #include <asm/asm-offsets.h>
12 #include <asm/rwlock.h>
17 .macro coreslot_loadaddr reg:req
18 \reg\().l = _corelock;
19 \reg\().h = _corelock;
22 .macro safe_testset addr:req, scratch:req
33 * r0 = address of atomic data to flush and invalidate (32bit).
35 * Clear interrupts and return the old mask.
36 * We assume that no atomic data can span cachelines.
47 if cc jump .Ldone_corelock;
56 ENDPROC(_get_core_lock)
59 * r0 = address of atomic data in uncacheable memory region (32bit).
61 * Clear interrupts and return the old mask.
65 ENTRY(_get_core_lock_noflush)
68 .Lretry_corelock_noflush:
70 if cc jump .Ldone_corelock_noflush;
72 jump .Lretry_corelock_noflush
73 .Ldone_corelock_noflush:
75 ENDPROC(_get_core_lock_noflush)
78 * r0 = interrupt mask to restore.
79 * r1 = address of atomic data to flush and invalidate (32bit).
81 * Interrupts are masked on entry (see _get_core_lock).
85 /* Write-through cache assumed, so no flush needed here. */
92 ENDPROC(_put_core_lock)
94 #ifdef __ARCH_SYNC_CORE_DCACHE
96 ENTRY(___raw_smp_mark_barrier_asm)
102 call _get_core_lock_noflush;
105 * Calculate current core mask
112 * Set bit of other cores in barrier mask. Don't change current core bit.
114 p1.l = _barrier_mask;
115 p1.h = _barrier_mask;
133 ENDPROC(___raw_smp_mark_barrier_asm)
135 ENTRY(___raw_smp_check_barrier_asm)
141 call _get_core_lock_noflush;
144 * Calculate current core mask
151 * Clear current core bit in barrier mask if it is set.
153 p1.l = _barrier_mask;
154 p1.h = _barrier_mask;
167 * Invalidate the entire D-cache of current core.
170 call _resync_core_dcache
182 ENDPROC(___raw_smp_check_barrier_asm)
186 * r1 = address of atomic data
188 * Clobbers: r2:0, p1:0
190 _start_lock_coherent:
198 * Determine whether the atomic data was previously
199 * owned by another CPU (=r6).
207 r1 >>= 28; /* CPU fingerprints are stored in the high nibble. */
215 * Release the core lock now, but keep IRQs disabled while we are
216 * performing the remaining housekeeping chores for the current CPU.
218 coreslot_loadaddr p0;
223 * If another CPU has owned the same atomic section before us,
224 * then our D-cached copy of the shared data protected by the
225 * current spin/write_lock may be obsolete.
228 if cc jump .Lcache_synced
231 * Invalidate the entire D-cache of the current core.
234 call _resync_core_dcache
246 * r1 = address of atomic data
248 * Clobbers: r2:0, p1:0
263 #endif /* __ARCH_SYNC_CORE_DCACHE */
266 * r0 = &spinlock->lock
268 * Clobbers: r3:0, p1:0
270 ENTRY(___raw_spin_is_locked_asm)
275 cc = bittst( r3, 0 );
282 ENDPROC(___raw_spin_is_locked_asm)
285 * r0 = &spinlock->lock
287 * Clobbers: r3:0, p1:0
289 ENTRY(___raw_spin_lock_asm)
296 cc = bittst( r2, 0 );
297 if cc jump .Lbusy_spinlock
298 #ifdef __ARCH_SYNC_CORE_DCACHE
300 bitset ( r2, 0 ); /* Raise the lock bit. */
302 call _start_lock_coherent
312 /* We don't touch the atomic area if busy, so that flush
313 will behave like nop in _put_core_lock. */
317 jump .Lretry_spinlock
318 ENDPROC(___raw_spin_lock_asm)
321 * r0 = &spinlock->lock
323 * Clobbers: r3:0, p1:0
325 ENTRY(___raw_spin_trylock_asm)
331 cc = bittst( r3, 0 );
332 if cc jump .Lfailed_trylock
333 #ifdef __ARCH_SYNC_CORE_DCACHE
334 bitset ( r3, 0 ); /* Raise the lock bit. */
336 call _start_lock_coherent
350 ENDPROC(___raw_spin_trylock_asm)
353 * r0 = &spinlock->lock
355 * Clobbers: r2:0, p1:0
357 ENTRY(___raw_spin_unlock_asm)
365 #ifdef __ARCH_SYNC_CORE_DCACHE
366 call _end_lock_coherent
372 ENDPROC(___raw_spin_unlock_asm)
377 * Clobbers: r2:0, p1:0
379 ENTRY(___raw_read_lock_asm)
388 if cc jump .Lrdlock_failed
390 #ifdef __ARCH_SYNC_CORE_DCACHE
391 call _start_lock_coherent
409 if cc jump .Lrdlock_wait;
411 ENDPROC(___raw_read_lock_asm)
416 * Clobbers: r3:0, p1:0
418 ENTRY(___raw_read_trylock_asm)
424 if cc jump .Lfailed_tryrdlock;
428 #ifdef __ARCH_SYNC_CORE_DCACHE
429 call _start_lock_coherent
442 ENDPROC(___raw_read_trylock_asm)
447 * Note: Processing controlled by a reader lock should not have
448 * any side-effect on cache issues with the other core, so we
449 * just release the core lock and exit (no _end_lock_coherent).
451 * Clobbers: r3:0, p1:0
453 ENTRY(___raw_read_unlock_asm)
464 ENDPROC(___raw_read_unlock_asm)
469 * Clobbers: r3:0, p1:0
471 ENTRY(___raw_write_lock_asm)
473 r3.l = lo(RW_LOCK_BIAS);
474 r3.h = hi(RW_LOCK_BIAS);
480 #ifdef __ARCH_SYNC_CORE_DCACHE
488 if !cc jump .Lwrlock_wait
491 #ifdef __ARCH_SYNC_CORE_DCACHE
492 call _start_lock_coherent
506 #ifdef __ARCH_SYNC_CORE_DCACHE
511 if !cc jump .Lwrlock_wait;
513 ENDPROC(___raw_write_lock_asm)
518 * Clobbers: r3:0, p1:0
520 ENTRY(___raw_write_trylock_asm)
525 r2.l = lo(RW_LOCK_BIAS);
526 r2.h = hi(RW_LOCK_BIAS);
528 if !cc jump .Lfailed_trywrlock;
529 #ifdef __ARCH_SYNC_CORE_DCACHE
537 #ifdef __ARCH_SYNC_CORE_DCACHE
538 call _start_lock_coherent
552 ENDPROC(___raw_write_trylock_asm)
557 * Clobbers: r3:0, p1:0
559 ENTRY(___raw_write_unlock_asm)
561 r3.l = lo(RW_LOCK_BIAS);
562 r3.h = hi(RW_LOCK_BIAS);
569 #ifdef __ARCH_SYNC_CORE_DCACHE
570 call _end_lock_coherent
576 ENDPROC(___raw_write_unlock_asm)
582 * Add a signed value to a 32bit word and return the new value atomically.
583 * Clobbers: r3:0, p1:0
585 ENTRY(___raw_atomic_update_asm)
598 ENDPROC(___raw_atomic_update_asm)
604 * Clear the mask bits from a 32bit word and return the old 32bit value
606 * Clobbers: r3:0, p1:0
608 ENTRY(___raw_atomic_clear_asm)
622 ENDPROC(___raw_atomic_clear_asm)
628 * Set the mask bits into a 32bit word and return the old 32bit value
630 * Clobbers: r3:0, p1:0
632 ENTRY(___raw_atomic_set_asm)
646 ENDPROC(___raw_atomic_set_asm)
652 * XOR the mask bits with a 32bit word and return the old 32bit value
654 * Clobbers: r3:0, p1:0
656 ENTRY(___raw_atomic_xor_asm)
670 ENDPROC(___raw_atomic_xor_asm)
676 * Perform a logical AND between the mask bits and a 32bit word, and
677 * return the masked value. We need this on this architecture in
678 * order to invalidate the local cache before testing.
680 * Clobbers: r3:0, p1:0
682 ENTRY(___raw_atomic_test_asm)
685 r1 = -L1_CACHE_BYTES;
693 ENDPROC(___raw_atomic_test_asm)
699 * Swap *ptr with value and return the old 32bit value atomically.
700 * Clobbers: r3:0, p1:0
702 #define __do_xchg(src, dst) \
706 call _get_core_lock; \
711 call _put_core_lock; \
716 ENTRY(___raw_xchg_1_asm)
717 __do_xchg(b[p1] (z), b[p1])
718 ENDPROC(___raw_xchg_1_asm)
720 ENTRY(___raw_xchg_2_asm)
721 __do_xchg(w[p1] (z), w[p1])
722 ENDPROC(___raw_xchg_2_asm)
724 ENTRY(___raw_xchg_4_asm)
725 __do_xchg([p1], [p1])
726 ENDPROC(___raw_xchg_4_asm)
733 * Swap *ptr with new if *ptr == old and return the previous *ptr
736 * Clobbers: r3:0, p1:0
738 #define __do_cmpxchg(src, dst) \
744 call _get_core_lock; \
751 call _put_core_lock; \
757 ENTRY(___raw_cmpxchg_1_asm)
758 __do_cmpxchg(b[p1] (z), b[p1])
759 ENDPROC(___raw_cmpxchg_1_asm)
761 ENTRY(___raw_cmpxchg_2_asm)
762 __do_cmpxchg(w[p1] (z), w[p1])
763 ENDPROC(___raw_cmpxchg_2_asm)
765 ENTRY(___raw_cmpxchg_4_asm)
766 __do_cmpxchg([p1], [p1])
767 ENDPROC(___raw_cmpxchg_4_asm)
773 * Set a bit in a 32bit word and return the old 32bit value atomically.
774 * Clobbers: r3:0, p1:0
776 ENTRY(___raw_bit_set_asm)
780 jump ___raw_atomic_set_asm
781 ENDPROC(___raw_bit_set_asm)
787 * Clear a bit in a 32bit word and return the old 32bit value atomically.
788 * Clobbers: r3:0, p1:0
790 ENTRY(___raw_bit_clear_asm)
794 jump ___raw_atomic_clear_asm
795 ENDPROC(___raw_bit_clear_asm)
801 * Toggle a bit in a 32bit word and return the old 32bit value atomically.
802 * Clobbers: r3:0, p1:0
804 ENTRY(___raw_bit_toggle_asm)
808 jump ___raw_atomic_xor_asm
809 ENDPROC(___raw_bit_toggle_asm)
815 * Test-and-set a bit in a 32bit word and return the old bit value atomically.
816 * Clobbers: r3:0, p1:0
818 ENTRY(___raw_bit_test_set_asm)
821 call ___raw_bit_set_asm
832 ENDPROC(___raw_bit_test_set_asm)
838 * Test-and-clear a bit in a 32bit word and return the old bit value atomically.
839 * Clobbers: r3:0, p1:0
841 ENTRY(___raw_bit_test_clear_asm)
844 call ___raw_bit_clear_asm
855 ENDPROC(___raw_bit_test_clear_asm)
861 * Test-and-toggle a bit in a 32bit word,
862 * and return the old bit value atomically.
863 * Clobbers: r3:0, p1:0
865 ENTRY(___raw_bit_test_toggle_asm)
868 call ___raw_bit_toggle_asm
879 ENDPROC(___raw_bit_test_toggle_asm)
885 * Test a bit in a 32bit word and return its value.
886 * We need this on this architecture in order to invalidate
887 * the local cache before testing.
889 * Clobbers: r3:0, p1:0
891 ENTRY(___raw_bit_test_asm)
895 jump ___raw_atomic_test_asm
896 ENDPROC(___raw_bit_test_asm)
901 * Fetch and return an uncached 32bit value.
903 * Clobbers: r2:0, p1:0
905 ENTRY(___raw_uncached_fetch_asm)
907 r1 = -L1_CACHE_BYTES;
914 ENDPROC(___raw_uncached_fetch_asm)