Merge branch 'for-3.14-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

[~andy/linux] / arch / arm64 / include / asm / atomic.h

diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
index 01de5aaa3edc112a42548a1b3906e4353bcce949..0237f0867e377a1acc4f3840e1a9307cff6a3fe8 100644 (file)
--- a/arch/arm64/include/asm/atomic.h
+++ b/arch/arm64/include/asm/atomic.h
@@ -54,8 +54,7 @@ static inline void atomic_add(int i, atomic_t *v)
 "      stxr    %w1, %w0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
-       : "Ir" (i)
-       : "cc");
+       : "Ir" (i));
 }
 
 static inline int atomic_add_return(int i, atomic_t *v)
@@ -64,14 +63,15 @@ static inline int atomic_add_return(int i, atomic_t *v)
        int result;
 
        asm volatile("// atomic_add_return\n"
-"1:    ldaxr   %w0, %2\n"
+"1:    ldxr    %w0, %2\n"
 "      add     %w0, %w0, %w3\n"
 "      stlxr   %w1, %w0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        : "Ir" (i)
-       : "cc", "memory");
+       : "memory");
 
+       smp_mb();
        return result;
 }
 
@@ -86,8 +86,7 @@ static inline void atomic_sub(int i, atomic_t *v)
 "      stxr    %w1, %w0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
-       : "Ir" (i)
-       : "cc");
+       : "Ir" (i));
 }
 
 static inline int atomic_sub_return(int i, atomic_t *v)
@@ -96,14 +95,15 @@ static inline int atomic_sub_return(int i, atomic_t *v)
        int result;
 
        asm volatile("// atomic_sub_return\n"
-"1:    ldaxr   %w0, %2\n"
+"1:    ldxr    %w0, %2\n"
 "      sub     %w0, %w0, %w3\n"
 "      stlxr   %w1, %w0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        : "Ir" (i)
-       : "cc", "memory");
+       : "memory");
 
+       smp_mb();
        return result;
 }
 
@@ -112,17 +112,20 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
        unsigned long tmp;
        int oldval;
 
+       smp_mb();
+
        asm volatile("// atomic_cmpxchg\n"
-"1:    ldaxr   %w1, %2\n"
+"1:    ldxr    %w1, %2\n"
 "      cmp     %w1, %w3\n"
 "      b.ne    2f\n"
-"      stlxr   %w0, %w4, %2\n"
+"      stxr    %w0, %w4, %2\n"
 "      cbnz    %w0, 1b\n"
 "2:"
        : "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
        : "Ir" (old), "r" (new)
-       : "cc", "memory");
+       : "cc");
 
+       smp_mb();
        return oldval;
 }
 
@@ -173,8 +176,7 @@ static inline void atomic64_add(u64 i, atomic64_t *v)
 "      stxr    %w1, %0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
-       : "Ir" (i)
-       : "cc");
+       : "Ir" (i));
 }
 
 static inline long atomic64_add_return(long i, atomic64_t *v)
@@ -183,14 +185,15 @@ static inline long atomic64_add_return(long i, atomic64_t *v)
        unsigned long tmp;
 
        asm volatile("// atomic64_add_return\n"
-"1:    ldaxr   %0, %2\n"
+"1:    ldxr    %0, %2\n"
 "      add     %0, %0, %3\n"
 "      stlxr   %w1, %0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        : "Ir" (i)
-       : "cc", "memory");
+       : "memory");
 
+       smp_mb();
        return result;
 }
 
@@ -205,8 +208,7 @@ static inline void atomic64_sub(u64 i, atomic64_t *v)
 "      stxr    %w1, %0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
-       : "Ir" (i)
-       : "cc");
+       : "Ir" (i));
 }
 
 static inline long atomic64_sub_return(long i, atomic64_t *v)
@@ -215,14 +217,15 @@ static inline long atomic64_sub_return(long i, atomic64_t *v)
        unsigned long tmp;
 
        asm volatile("// atomic64_sub_return\n"
-"1:    ldaxr   %0, %2\n"
+"1:    ldxr    %0, %2\n"
 "      sub     %0, %0, %3\n"
 "      stlxr   %w1, %0, %2\n"
 "      cbnz    %w1, 1b"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        : "Ir" (i)
-       : "cc", "memory");
+       : "memory");
 
+       smp_mb();
        return result;
 }
 
@@ -231,17 +234,20 @@ static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
        long oldval;
        unsigned long res;
 
+       smp_mb();
+
        asm volatile("// atomic64_cmpxchg\n"
-"1:    ldaxr   %1, %2\n"
+"1:    ldxr    %1, %2\n"
 "      cmp     %1, %3\n"
 "      b.ne    2f\n"
-"      stlxr   %w0, %4, %2\n"
+"      stxr    %w0, %4, %2\n"
 "      cbnz    %w0, 1b\n"
 "2:"
        : "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
        : "Ir" (old), "r" (new)
-       : "cc", "memory");
+       : "cc");
 
+       smp_mb();
        return oldval;
 }
 
@@ -253,11 +259,12 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
        unsigned long tmp;
 
        asm volatile("// atomic64_dec_if_positive\n"
-"1:    ldaxr   %0, %2\n"
+"1:    ldxr    %0, %2\n"
 "      subs    %0, %0, #1\n"
 "      b.mi    2f\n"
 "      stlxr   %w1, %0, %2\n"
 "      cbnz    %w1, 1b\n"
+"      dmb     ish\n"
 "2:"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        :