2 * Copyright 2007-2008 Analog Devices Inc.
3 * Philippe Gerum <rpm@xenomai.org>
5 * Licensed under the GPL-2 or later.
8 #include <linux/linkage.h>
9 #include <asm/blackfin.h>
10 #include <asm/cache.h>
11 #include <asm/asm-offsets.h>
12 #include <asm/rwlock.h>
17 .macro coreslot_loadaddr reg:req
18 \reg\().l = _corelock;
19 \reg\().h = _corelock;
22 .macro safe_testset addr:req, scratch:req
33 * r0 = address of atomic data to flush and invalidate (32bit).
35 * Clear interrupts and return the old mask.
36 * We assume that no atomic data can span cachelines.
47 if cc jump .Ldone_corelock;
52 /* flush core internal write buffer before invalidate dcache */
57 ENDPROC(_get_core_lock)
60 * r0 = address of atomic data in uncacheable memory region (32bit).
62 * Clear interrupts and return the old mask.
66 ENTRY(_get_core_lock_noflush)
69 .Lretry_corelock_noflush:
71 if cc jump .Ldone_corelock_noflush;
73 jump .Lretry_corelock_noflush
74 .Ldone_corelock_noflush:
76 ENDPROC(_get_core_lock_noflush)
79 * r0 = interrupt mask to restore.
80 * r1 = address of atomic data to flush and invalidate (32bit).
82 * Interrupts are masked on entry (see _get_core_lock).
86 /* Write-through cache assumed, so no flush needed here. */
93 ENDPROC(_put_core_lock)
95 #ifdef __ARCH_SYNC_CORE_DCACHE
97 ENTRY(___raw_smp_mark_barrier_asm)
103 call _get_core_lock_noflush;
106 * Calculate current core mask
113 * Set bit of other cores in barrier mask. Don't change current core bit.
115 p1.l = _barrier_mask;
116 p1.h = _barrier_mask;
134 ENDPROC(___raw_smp_mark_barrier_asm)
136 ENTRY(___raw_smp_check_barrier_asm)
142 call _get_core_lock_noflush;
145 * Calculate current core mask
152 * Clear current core bit in barrier mask if it is set.
154 p1.l = _barrier_mask;
155 p1.h = _barrier_mask;
168 * Invalidate the entire D-cache of current core.
171 call _resync_core_dcache
183 ENDPROC(___raw_smp_check_barrier_asm)
187 * r1 = address of atomic data
189 * Clobbers: r2:0, p1:0
191 _start_lock_coherent:
199 * Determine whether the atomic data was previously
200 * owned by another CPU (=r6).
208 r1 >>= 28; /* CPU fingerprints are stored in the high nibble. */
216 * Release the core lock now, but keep IRQs disabled while we are
217 * performing the remaining housekeeping chores for the current CPU.
219 coreslot_loadaddr p0;
224 * If another CPU has owned the same atomic section before us,
225 * then our D-cached copy of the shared data protected by the
226 * current spin/write_lock may be obsolete.
229 if cc jump .Lcache_synced
232 * Invalidate the entire D-cache of the current core.
235 call _resync_core_dcache
247 * r1 = address of atomic data
249 * Clobbers: r2:0, p1:0
264 #endif /* __ARCH_SYNC_CORE_DCACHE */
267 * r0 = &spinlock->lock
269 * Clobbers: r3:0, p1:0
271 ENTRY(___raw_spin_is_locked_asm)
276 cc = bittst( r3, 0 );
283 ENDPROC(___raw_spin_is_locked_asm)
286 * r0 = &spinlock->lock
288 * Clobbers: r3:0, p1:0
290 ENTRY(___raw_spin_lock_asm)
297 cc = bittst( r2, 0 );
298 if cc jump .Lbusy_spinlock
299 #ifdef __ARCH_SYNC_CORE_DCACHE
301 bitset ( r2, 0 ); /* Raise the lock bit. */
303 call _start_lock_coherent
313 /* We don't touch the atomic area if busy, so that flush
314 will behave like nop in _put_core_lock. */
318 jump .Lretry_spinlock
319 ENDPROC(___raw_spin_lock_asm)
322 * r0 = &spinlock->lock
324 * Clobbers: r3:0, p1:0
326 ENTRY(___raw_spin_trylock_asm)
332 cc = bittst( r3, 0 );
333 if cc jump .Lfailed_trylock
334 #ifdef __ARCH_SYNC_CORE_DCACHE
335 bitset ( r3, 0 ); /* Raise the lock bit. */
337 call _start_lock_coherent
351 ENDPROC(___raw_spin_trylock_asm)
354 * r0 = &spinlock->lock
356 * Clobbers: r2:0, p1:0
358 ENTRY(___raw_spin_unlock_asm)
366 #ifdef __ARCH_SYNC_CORE_DCACHE
367 call _end_lock_coherent
373 ENDPROC(___raw_spin_unlock_asm)
378 * Clobbers: r2:0, p1:0
380 ENTRY(___raw_read_lock_asm)
389 if cc jump .Lrdlock_failed
391 #ifdef __ARCH_SYNC_CORE_DCACHE
392 call _start_lock_coherent
410 if cc jump .Lrdlock_wait;
412 ENDPROC(___raw_read_lock_asm)
417 * Clobbers: r3:0, p1:0
419 ENTRY(___raw_read_trylock_asm)
425 if cc jump .Lfailed_tryrdlock;
429 #ifdef __ARCH_SYNC_CORE_DCACHE
430 call _start_lock_coherent
443 ENDPROC(___raw_read_trylock_asm)
448 * Note: Processing controlled by a reader lock should not have
449 * any side-effect on cache issues with the other core, so we
450 * just release the core lock and exit (no _end_lock_coherent).
452 * Clobbers: r3:0, p1:0
454 ENTRY(___raw_read_unlock_asm)
465 ENDPROC(___raw_read_unlock_asm)
470 * Clobbers: r3:0, p1:0
472 ENTRY(___raw_write_lock_asm)
474 r3.l = lo(RW_LOCK_BIAS);
475 r3.h = hi(RW_LOCK_BIAS);
481 #ifdef __ARCH_SYNC_CORE_DCACHE
489 if !cc jump .Lwrlock_wait
492 #ifdef __ARCH_SYNC_CORE_DCACHE
493 call _start_lock_coherent
507 #ifdef __ARCH_SYNC_CORE_DCACHE
512 if !cc jump .Lwrlock_wait;
514 ENDPROC(___raw_write_lock_asm)
519 * Clobbers: r3:0, p1:0
521 ENTRY(___raw_write_trylock_asm)
526 r2.l = lo(RW_LOCK_BIAS);
527 r2.h = hi(RW_LOCK_BIAS);
529 if !cc jump .Lfailed_trywrlock;
530 #ifdef __ARCH_SYNC_CORE_DCACHE
538 #ifdef __ARCH_SYNC_CORE_DCACHE
539 call _start_lock_coherent
553 ENDPROC(___raw_write_trylock_asm)
558 * Clobbers: r3:0, p1:0
560 ENTRY(___raw_write_unlock_asm)
562 r3.l = lo(RW_LOCK_BIAS);
563 r3.h = hi(RW_LOCK_BIAS);
570 #ifdef __ARCH_SYNC_CORE_DCACHE
571 call _end_lock_coherent
577 ENDPROC(___raw_write_unlock_asm)
583 * Add a signed value to a 32bit word and return the new value atomically.
584 * Clobbers: r3:0, p1:0
586 ENTRY(___raw_atomic_update_asm)
599 ENDPROC(___raw_atomic_update_asm)
605 * Clear the mask bits from a 32bit word and return the old 32bit value
607 * Clobbers: r3:0, p1:0
609 ENTRY(___raw_atomic_clear_asm)
623 ENDPROC(___raw_atomic_clear_asm)
629 * Set the mask bits into a 32bit word and return the old 32bit value
631 * Clobbers: r3:0, p1:0
633 ENTRY(___raw_atomic_set_asm)
647 ENDPROC(___raw_atomic_set_asm)
653 * XOR the mask bits with a 32bit word and return the old 32bit value
655 * Clobbers: r3:0, p1:0
657 ENTRY(___raw_atomic_xor_asm)
671 ENDPROC(___raw_atomic_xor_asm)
677 * Perform a logical AND between the mask bits and a 32bit word, and
678 * return the masked value. We need this on this architecture in
679 * order to invalidate the local cache before testing.
681 * Clobbers: r3:0, p1:0
683 ENTRY(___raw_atomic_test_asm)
686 r1 = -L1_CACHE_BYTES;
689 /* flush core internal write buffer before invalidate dcache */
696 ENDPROC(___raw_atomic_test_asm)
702 * Swap *ptr with value and return the old 32bit value atomically.
703 * Clobbers: r3:0, p1:0
705 #define __do_xchg(src, dst) \
709 call _get_core_lock; \
714 call _put_core_lock; \
719 ENTRY(___raw_xchg_1_asm)
720 __do_xchg(b[p1] (z), b[p1])
721 ENDPROC(___raw_xchg_1_asm)
723 ENTRY(___raw_xchg_2_asm)
724 __do_xchg(w[p1] (z), w[p1])
725 ENDPROC(___raw_xchg_2_asm)
727 ENTRY(___raw_xchg_4_asm)
728 __do_xchg([p1], [p1])
729 ENDPROC(___raw_xchg_4_asm)
736 * Swap *ptr with new if *ptr == old and return the previous *ptr
739 * Clobbers: r3:0, p1:0
741 #define __do_cmpxchg(src, dst) \
747 call _get_core_lock; \
754 call _put_core_lock; \
760 ENTRY(___raw_cmpxchg_1_asm)
761 __do_cmpxchg(b[p1] (z), b[p1])
762 ENDPROC(___raw_cmpxchg_1_asm)
764 ENTRY(___raw_cmpxchg_2_asm)
765 __do_cmpxchg(w[p1] (z), w[p1])
766 ENDPROC(___raw_cmpxchg_2_asm)
768 ENTRY(___raw_cmpxchg_4_asm)
769 __do_cmpxchg([p1], [p1])
770 ENDPROC(___raw_cmpxchg_4_asm)
776 * Set a bit in a 32bit word and return the old 32bit value atomically.
777 * Clobbers: r3:0, p1:0
779 ENTRY(___raw_bit_set_asm)
783 jump ___raw_atomic_set_asm
784 ENDPROC(___raw_bit_set_asm)
790 * Clear a bit in a 32bit word and return the old 32bit value atomically.
791 * Clobbers: r3:0, p1:0
793 ENTRY(___raw_bit_clear_asm)
797 jump ___raw_atomic_clear_asm
798 ENDPROC(___raw_bit_clear_asm)
804 * Toggle a bit in a 32bit word and return the old 32bit value atomically.
805 * Clobbers: r3:0, p1:0
807 ENTRY(___raw_bit_toggle_asm)
811 jump ___raw_atomic_xor_asm
812 ENDPROC(___raw_bit_toggle_asm)
818 * Test-and-set a bit in a 32bit word and return the old bit value atomically.
819 * Clobbers: r3:0, p1:0
821 ENTRY(___raw_bit_test_set_asm)
824 call ___raw_bit_set_asm
835 ENDPROC(___raw_bit_test_set_asm)
841 * Test-and-clear a bit in a 32bit word and return the old bit value atomically.
842 * Clobbers: r3:0, p1:0
844 ENTRY(___raw_bit_test_clear_asm)
847 call ___raw_bit_clear_asm
858 ENDPROC(___raw_bit_test_clear_asm)
864 * Test-and-toggle a bit in a 32bit word,
865 * and return the old bit value atomically.
866 * Clobbers: r3:0, p1:0
868 ENTRY(___raw_bit_test_toggle_asm)
871 call ___raw_bit_toggle_asm
882 ENDPROC(___raw_bit_test_toggle_asm)
888 * Test a bit in a 32bit word and return its value.
889 * We need this on this architecture in order to invalidate
890 * the local cache before testing.
892 * Clobbers: r3:0, p1:0
894 ENTRY(___raw_bit_test_asm)
898 jump ___raw_atomic_test_asm
899 ENDPROC(___raw_bit_test_asm)
904 * Fetch and return an uncached 32bit value.
906 * Clobbers: r2:0, p1:0
908 ENTRY(___raw_uncached_fetch_asm)
910 r1 = -L1_CACHE_BYTES;
913 /* flush core internal write buffer before invalidate dcache */
919 ENDPROC(___raw_uncached_fetch_asm)