Pileus Git - ~andy/linux/blob - include/asm-x86/spinlock_32.h

   1 #ifndef __ASM_SPINLOCK_H
   2 #define __ASM_SPINLOCK_H
   3
   4 #include <asm/atomic.h>
   5 #include <asm/rwlock.h>
   6 #include <asm/page.h>
   7 #include <asm/processor.h>
   8
   9 /*
  10  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  11  *
  12  * Simple spin lock operations.  There are two variants, one clears IRQ's
  13  * on the local processor, one does not.
  14  *
  15  * We make no fairness assumptions. They have a cost.
  16  *
  17  * (the type definitions are in asm/spinlock_types.h)
  18  */
  19
  20 static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
  21 {
  22         return *(volatile signed char *)(&(lock)->slock) <= 0;
  23 }
  24
  25 static inline void __raw_spin_lock(raw_spinlock_t *lock)
  26 {
  27         asm volatile(
  28                 "\n1:\t"
  29                 LOCK_PREFIX " ; decb %0\n\t"
  30                 "jns 3f\n"
  31                 "2:\t"
  32                 "rep;nop\n\t"
  33                 "cmpb $0,%0\n\t"
  34                 "jle 2b\n\t"
  35                 "jmp 1b\n"
  36                 "3:\n\t"
  37                 : "+m" (lock->slock) : : "memory");
  38 }
  39
  40 /*
  41  * It is easier for the lock validator if interrupts are not re-enabled
  42  * in the middle of a lock-acquire. This is a performance feature anyway
  43  * so we turn it off:
  44  *
  45  * NOTE: there's an irqs-on section here, which normally would have to be
  46  * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant.
  47  */
  48 #ifndef CONFIG_PROVE_LOCKING
  49 static inline void __raw_spin_lock_flags(raw_spinlock_t *lock,
  50                                          unsigned long flags)
  51 {
  52         asm volatile(
  53                 "\n1:\t"
  54                 LOCK_PREFIX " ; decb %[slock]\n\t"
  55                 "jns 5f\n"
  56                 "testl $0x200, %[flags]\n\t"
  57                 "jz 4f\n\t"
  58                 STI_STRING "\n"
  59                 "3:\t"
  60                 "rep;nop\n\t"
  61                 "cmpb $0, %[slock]\n\t"
  62                 "jle 3b\n\t"
  63                 CLI_STRING "\n\t"
  64                 "jmp 1b\n"
  65                 "4:\t"
  66                 "rep;nop\n\t"
  67                 "cmpb $0, %[slock]\n\t"
  68                 "jg 1b\n\t"
  69                 "jmp 4b\n"
  70                 "5:\n\t"
  71                 : [slock] "+m" (lock->slock)
  72                 : [flags] "r" (flags)
  73                   CLI_STI_INPUT_ARGS
  74                 : "memory" CLI_STI_CLOBBERS);
  75 }
  76 #endif
  77
  78 static inline int __raw_spin_trylock(raw_spinlock_t *lock)
  79 {
  80         signed char oldval;
  81
  82         asm volatile(
  83                 "xchgb %b0,%1"
  84                 :"=q" (oldval), "+m" (lock->slock)
  85                 :"0" (0) : "memory");
  86
  87         return oldval > 0;
  88 }
  89
  90 /*
  91  * __raw_spin_unlock based on writing $1 to the low byte.
  92  * This method works. Despite all the confusion.
  93  * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there)
  94  * (PPro errata 66, 92)
  95  */
  96
  97 #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
  98
  99 static inline void __raw_spin_unlock(raw_spinlock_t *lock)
 100 {
 101         asm volatile("movb $1,%0" : "=m" (lock->slock) :: "memory");
 102 }
 103
 104 #else
 105
 106 static inline void __raw_spin_unlock(raw_spinlock_t *lock)
 107 {
 108         unsigned char oldval = 1;
 109
 110         asm volatile("xchgb %b0, %1"
 111                      : "=q" (oldval), "+m" (lock->slock)
 112                      : "0" (oldval) : "memory");
 113 }
 114
 115 #endif
 116
 117 static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock)
 118 {
 119         while (__raw_spin_is_locked(lock))
 120                 cpu_relax();
 121 }
 122
 123 /*
 124  * Read-write spinlocks, allowing multiple readers
 125  * but only one writer.
 126  *
 127  * NOTE! it is quite common to have readers in interrupts
 128  * but no interrupt writers. For those circumstances we
 129  * can "mix" irq-safe locks - any writer needs to get a
 130  * irq-safe write-lock, but readers can get non-irqsafe
 131  * read-locks.
 132  *
 133  * On x86, we implement read-write locks as a 32-bit counter
 134  * with the high bit (sign) being the "contended" bit.
 135  */
 136
 137 static inline int __raw_read_can_lock(raw_rwlock_t *lock)
 138 {
 139         return (int)(lock)->lock > 0;
 140 }
 141
 142 static inline int __raw_write_can_lock(raw_rwlock_t *lock)
 143 {
 144         return (lock)->lock == RW_LOCK_BIAS;
 145 }
 146
 147 static inline void __raw_read_lock(raw_rwlock_t *rw)
 148 {
 149         asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t"
 150                      "jns 1f\n"
 151                      "call __read_lock_failed\n\t"
 152                      "1:\n"
 153                      ::"a" (rw) : "memory");
 154 }
 155
 156 static inline void __raw_write_lock(raw_rwlock_t *rw)
 157 {
 158         asm volatile(LOCK_PREFIX " subl %1,(%0)\n\t"
 159                      "jz 1f\n"
 160                      "call __write_lock_failed\n\t"
 161                      "1:\n"
 162                      ::"a" (rw), "i" (RW_LOCK_BIAS) : "memory");
 163 }
 164
 165 static inline int __raw_read_trylock(raw_rwlock_t *lock)
 166 {
 167         atomic_t *count = (atomic_t *)lock;
 168
 169         atomic_dec(count);
 170         if (atomic_read(count) >= 0)
 171                 return 1;
 172         atomic_inc(count);
 173         return 0;
 174 }
 175
 176 static inline int __raw_write_trylock(raw_rwlock_t *lock)
 177 {
 178         atomic_t *count = (atomic_t *)lock;
 179
 180         if (atomic_sub_and_test(RW_LOCK_BIAS, count))
 181                 return 1;
 182         atomic_add(RW_LOCK_BIAS, count);
 183         return 0;
 184 }
 185
 186 static inline void __raw_read_unlock(raw_rwlock_t *rw)
 187 {
 188         asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory");
 189 }
 190
 191 static inline void __raw_write_unlock(raw_rwlock_t *rw)
 192 {
 193         asm volatile(LOCK_PREFIX "addl %1, %0"
 194                      : "+m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory");
 195 }
 196
 197 #define _raw_spin_relax(lock)   cpu_relax()
 198 #define _raw_read_relax(lock)   cpu_relax()
 199 #define _raw_write_relax(lock)  cpu_relax()
 200
 201 #endif /* __ASM_SPINLOCK_H */