diff options
Diffstat (limited to 'arch/loongarch/include/asm/cmpxchg.h')
| -rw-r--r-- | arch/loongarch/include/asm/cmpxchg.h | 107 |
1 files changed, 96 insertions, 11 deletions
diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h index 979fde61bba8..909f9274fe71 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -8,18 +8,35 @@ #include <linux/bits.h> #include <linux/build_bug.h> #include <asm/barrier.h> +#include <asm/cpu-features.h> -#define __xchg_asm(amswap_db, m, val) \ +#define __xchg_amo_asm(amswap_db, m, val) \ ({ \ - __typeof(val) __ret; \ + __typeof(val) __ret; \ \ - __asm__ __volatile__ ( \ - " "amswap_db" %1, %z2, %0 \n" \ - : "+ZB" (*m), "=&r" (__ret) \ - : "Jr" (val) \ - : "memory"); \ + __asm__ __volatile__ ( \ + " "amswap_db" %1, %z2, %0 \n" \ + : "+ZB" (*m), "=&r" (__ret) \ + : "Jr" (val) \ + : "memory"); \ \ - __ret; \ + __ret; \ +}) + +#define __xchg_llsc_asm(ld, st, m, val) \ +({ \ + __typeof(val) __ret, __tmp; \ + \ + asm volatile ( \ + "1: ll.w %0, %3 \n" \ + " move %1, %z4 \n" \ + " sc.w %1, %2 \n" \ + " beqz %1, 1b \n" \ + : "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \ + : "ZC" (*m), "Jr" (val) \ + : "memory"); \ + \ + __ret; \ }) static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val, @@ -67,13 +84,23 @@ __arch_xchg(volatile void *ptr, unsigned long x, int size) switch (size) { case 1: case 2: - return __xchg_small(ptr, x, size); + return __xchg_small((volatile void *)ptr, x, size); case 4: - return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); +#ifdef CONFIG_CPU_HAS_AMO + return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); +#else + return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x); +#endif /* CONFIG_CPU_HAS_AMO */ +#ifdef CONFIG_64BIT case 8: - return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); +#ifdef CONFIG_CPU_HAS_AMO + return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); +#else + return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x); +#endif /* CONFIG_CPU_HAS_AMO */ +#endif /* CONFIG_64BIT */ default: BUILD_BUG(); @@ -210,6 +237,64 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg((ptr), (o), (n)); \ }) + +#ifdef CONFIG_AS_HAS_SCQ_EXTENSION + +union __u128_halves { + u128 full; + struct { + u64 low; + u64 high; + }; +}; + +#define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ) + +#define __arch_cmpxchg128(ptr, old, new, llsc_mb) \ +({ \ + union __u128_halves __old, __new, __ret; \ + volatile u64 *__ptr = (volatile u64 *)(ptr); \ + \ + __old.full = (old); \ + __new.full = (new); \ + \ + __asm__ __volatile__( \ + "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \ + llsc_mb \ + " ld.d %1, %4 # 128-bit cmpxchg high \n" \ + " move $t0, %0 \n" \ + " move $t1, %1 \n" \ + " bne %0, %z5, 2f \n" \ + " bne %1, %z6, 2f \n" \ + " move $t0, %z7 \n" \ + " move $t1, %z8 \n" \ + "2: sc.q $t0, $t1, %2 \n" \ + " beqz $t0, 1b \n" \ + llsc_mb \ + : "=&r" (__ret.low), "=&r" (__ret.high) \ + : "r" (__ptr), \ + "ZC" (__ptr[0]), "m" (__ptr[1]), \ + "Jr" (__old.low), "Jr" (__old.high), \ + "Jr" (__new.low), "Jr" (__new.high) \ + : "t0", "t1", "memory"); \ + \ + __ret.full; \ +}) + +#define arch_cmpxchg128(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \ +}) + +#define arch_cmpxchg128_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ + __arch_cmpxchg128(ptr, o, n, ""); \ +}) + +#endif /* CONFIG_AS_HAS_SCQ_EXTENSION */ + #else #include <asm-generic/cmpxchg-local.h> #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) |