mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-27 12:57:53 +00:00
fd2efaa4eb
Currently architectures can override __atomic_op_*() to define the barriers used before/after a relaxed atomic when used to build acquire/release/fence variants. This has the unfortunate property of requiring the architecture to define the full wrapper for the atomics, rather than just the barriers they care about, and gets in the way of generating atomics which can be easily read. Instead, this patch has architectures define an optional set of barriers: * __atomic_acquire_fence() * __atomic_release_fence() * __atomic_pre_full_fence() * __atomic_post_full_fence() ... which <linux/atomic.h> uses to build the wrappers. It would be nice if we could undef these, along with the __atomic_op_*() wrappers, but that would break the cmpxchg() wrappers, which are written in preprocessor. Undefs would have been nice, but alas. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Will Deacon <will.deacon@arm.com> Cc: Andrea Parri <parri.andrea@gmail.com> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: andy.shevchenko@gmail.com Cc: arnd@arndb.de Cc: aryabinin@virtuozzo.com Cc: catalin.marinas@arm.com Cc: dvyukov@google.com Cc: glider@google.com Cc: linux-arm-kernel@lists.infradead.org Cc: peter@hurleysoftware.com Link: http://lkml.kernel.org/r/20180716113017.3909-7-mark.rutland@arm.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
351 lines
10 KiB
C
351 lines
10 KiB
C
/*
|
|
* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
|
|
* Copyright (C) 2012 Regents of the University of California
|
|
* Copyright (C) 2017 SiFive
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public Licence
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the Licence, or (at your option) any later version.
|
|
*/
|
|
|
|
#ifndef _ASM_RISCV_ATOMIC_H
|
|
#define _ASM_RISCV_ATOMIC_H
|
|
|
|
#ifdef CONFIG_GENERIC_ATOMIC64
|
|
# include <asm-generic/atomic64.h>
|
|
#else
|
|
# if (__riscv_xlen < 64)
|
|
# error "64-bit atomics require XLEN to be at least 64"
|
|
# endif
|
|
#endif
|
|
|
|
#include <asm/cmpxchg.h>
|
|
#include <asm/barrier.h>
|
|
|
|
#define ATOMIC_INIT(i) { (i) }
|
|
|
|
#define __atomic_acquire_fence() \
|
|
__asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
|
|
|
|
#define __atomic_release_fence() \
|
|
__asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
|
|
|
|
static __always_inline int atomic_read(const atomic_t *v)
|
|
{
|
|
return READ_ONCE(v->counter);
|
|
}
|
|
static __always_inline void atomic_set(atomic_t *v, int i)
|
|
{
|
|
WRITE_ONCE(v->counter, i);
|
|
}
|
|
|
|
#ifndef CONFIG_GENERIC_ATOMIC64
|
|
#define ATOMIC64_INIT(i) { (i) }
|
|
static __always_inline long atomic64_read(const atomic64_t *v)
|
|
{
|
|
return READ_ONCE(v->counter);
|
|
}
|
|
static __always_inline void atomic64_set(atomic64_t *v, long i)
|
|
{
|
|
WRITE_ONCE(v->counter, i);
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* First, the atomic ops that have no ordering constraints and therefor don't
|
|
* have the AQ or RL bits set. These don't return anything, so there's only
|
|
* one version to worry about.
|
|
*/
|
|
#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
|
|
static __always_inline \
|
|
void atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
|
|
{ \
|
|
__asm__ __volatile__ ( \
|
|
" amo" #asm_op "." #asm_type " zero, %1, %0" \
|
|
: "+A" (v->counter) \
|
|
: "r" (I) \
|
|
: "memory"); \
|
|
} \
|
|
|
|
#ifdef CONFIG_GENERIC_ATOMIC64
|
|
#define ATOMIC_OPS(op, asm_op, I) \
|
|
ATOMIC_OP (op, asm_op, I, w, int, )
|
|
#else
|
|
#define ATOMIC_OPS(op, asm_op, I) \
|
|
ATOMIC_OP (op, asm_op, I, w, int, ) \
|
|
ATOMIC_OP (op, asm_op, I, d, long, 64)
|
|
#endif
|
|
|
|
ATOMIC_OPS(add, add, i)
|
|
ATOMIC_OPS(sub, add, -i)
|
|
ATOMIC_OPS(and, and, i)
|
|
ATOMIC_OPS( or, or, i)
|
|
ATOMIC_OPS(xor, xor, i)
|
|
|
|
#undef ATOMIC_OP
|
|
#undef ATOMIC_OPS
|
|
|
|
/*
|
|
* Atomic ops that have ordered, relaxed, acquire, and release variants.
|
|
* There's two flavors of these: the arithmatic ops have both fetch and return
|
|
* versions, while the logical ops only have fetch versions.
|
|
*/
|
|
#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
|
|
static __always_inline \
|
|
c_type atomic##prefix##_fetch_##op##_relaxed(c_type i, \
|
|
atomic##prefix##_t *v) \
|
|
{ \
|
|
register c_type ret; \
|
|
__asm__ __volatile__ ( \
|
|
" amo" #asm_op "." #asm_type " %1, %2, %0" \
|
|
: "+A" (v->counter), "=r" (ret) \
|
|
: "r" (I) \
|
|
: "memory"); \
|
|
return ret; \
|
|
} \
|
|
static __always_inline \
|
|
c_type atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
|
|
{ \
|
|
register c_type ret; \
|
|
__asm__ __volatile__ ( \
|
|
" amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
|
|
: "+A" (v->counter), "=r" (ret) \
|
|
: "r" (I) \
|
|
: "memory"); \
|
|
return ret; \
|
|
}
|
|
|
|
#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
|
|
static __always_inline \
|
|
c_type atomic##prefix##_##op##_return_relaxed(c_type i, \
|
|
atomic##prefix##_t *v) \
|
|
{ \
|
|
return atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
|
|
} \
|
|
static __always_inline \
|
|
c_type atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
|
|
{ \
|
|
return atomic##prefix##_fetch_##op(i, v) c_op I; \
|
|
}
|
|
|
|
#ifdef CONFIG_GENERIC_ATOMIC64
|
|
#define ATOMIC_OPS(op, asm_op, c_op, I) \
|
|
ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
|
|
ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
|
|
#else
|
|
#define ATOMIC_OPS(op, asm_op, c_op, I) \
|
|
ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
|
|
ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
|
|
ATOMIC_FETCH_OP( op, asm_op, I, d, long, 64) \
|
|
ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, long, 64)
|
|
#endif
|
|
|
|
ATOMIC_OPS(add, add, +, i)
|
|
ATOMIC_OPS(sub, add, +, -i)
|
|
|
|
#define atomic_add_return_relaxed atomic_add_return_relaxed
|
|
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
|
|
#define atomic_add_return atomic_add_return
|
|
#define atomic_sub_return atomic_sub_return
|
|
|
|
#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
|
|
#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
|
|
#define atomic_fetch_add atomic_fetch_add
|
|
#define atomic_fetch_sub atomic_fetch_sub
|
|
|
|
#ifndef CONFIG_GENERIC_ATOMIC64
|
|
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
|
|
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
|
|
#define atomic64_add_return atomic64_add_return
|
|
#define atomic64_sub_return atomic64_sub_return
|
|
|
|
#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
|
|
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
|
|
#define atomic64_fetch_add atomic64_fetch_add
|
|
#define atomic64_fetch_sub atomic64_fetch_sub
|
|
#endif
|
|
|
|
#undef ATOMIC_OPS
|
|
|
|
#ifdef CONFIG_GENERIC_ATOMIC64
|
|
#define ATOMIC_OPS(op, asm_op, I) \
|
|
ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
|
|
#else
|
|
#define ATOMIC_OPS(op, asm_op, I) \
|
|
ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
|
|
ATOMIC_FETCH_OP(op, asm_op, I, d, long, 64)
|
|
#endif
|
|
|
|
ATOMIC_OPS(and, and, i)
|
|
ATOMIC_OPS( or, or, i)
|
|
ATOMIC_OPS(xor, xor, i)
|
|
|
|
#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
|
|
#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
|
|
#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
|
|
#define atomic_fetch_and atomic_fetch_and
|
|
#define atomic_fetch_or atomic_fetch_or
|
|
#define atomic_fetch_xor atomic_fetch_xor
|
|
|
|
#ifndef CONFIG_GENERIC_ATOMIC64
|
|
#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
|
|
#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
|
|
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
|
|
#define atomic64_fetch_and atomic64_fetch_and
|
|
#define atomic64_fetch_or atomic64_fetch_or
|
|
#define atomic64_fetch_xor atomic64_fetch_xor
|
|
#endif
|
|
|
|
#undef ATOMIC_OPS
|
|
|
|
#undef ATOMIC_FETCH_OP
|
|
#undef ATOMIC_OP_RETURN
|
|
|
|
/* This is required to provide a full barrier on success. */
|
|
static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
|
|
{
|
|
int prev, rc;
|
|
|
|
__asm__ __volatile__ (
|
|
"0: lr.w %[p], %[c]\n"
|
|
" beq %[p], %[u], 1f\n"
|
|
" add %[rc], %[p], %[a]\n"
|
|
" sc.w.rl %[rc], %[rc], %[c]\n"
|
|
" bnez %[rc], 0b\n"
|
|
" fence rw, rw\n"
|
|
"1:\n"
|
|
: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
|
|
: [a]"r" (a), [u]"r" (u)
|
|
: "memory");
|
|
return prev;
|
|
}
|
|
#define atomic_fetch_add_unless atomic_fetch_add_unless
|
|
|
|
#ifndef CONFIG_GENERIC_ATOMIC64
|
|
static __always_inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
|
|
{
|
|
long prev, rc;
|
|
|
|
__asm__ __volatile__ (
|
|
"0: lr.d %[p], %[c]\n"
|
|
" beq %[p], %[u], 1f\n"
|
|
" add %[rc], %[p], %[a]\n"
|
|
" sc.d.rl %[rc], %[rc], %[c]\n"
|
|
" bnez %[rc], 0b\n"
|
|
" fence rw, rw\n"
|
|
"1:\n"
|
|
: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
|
|
: [a]"r" (a), [u]"r" (u)
|
|
: "memory");
|
|
return prev;
|
|
}
|
|
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
|
|
#endif
|
|
|
|
/*
|
|
* atomic_{cmp,}xchg is required to have exactly the same ordering semantics as
|
|
* {cmp,}xchg and the operations that return, so they need a full barrier.
|
|
*/
|
|
#define ATOMIC_OP(c_t, prefix, size) \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_xchg_relaxed(atomic##prefix##_t *v, c_t n) \
|
|
{ \
|
|
return __xchg_relaxed(&(v->counter), n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_xchg_acquire(atomic##prefix##_t *v, c_t n) \
|
|
{ \
|
|
return __xchg_acquire(&(v->counter), n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_xchg_release(atomic##prefix##_t *v, c_t n) \
|
|
{ \
|
|
return __xchg_release(&(v->counter), n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_xchg(atomic##prefix##_t *v, c_t n) \
|
|
{ \
|
|
return __xchg(&(v->counter), n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_cmpxchg_relaxed(atomic##prefix##_t *v, \
|
|
c_t o, c_t n) \
|
|
{ \
|
|
return __cmpxchg_relaxed(&(v->counter), o, n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_cmpxchg_acquire(atomic##prefix##_t *v, \
|
|
c_t o, c_t n) \
|
|
{ \
|
|
return __cmpxchg_acquire(&(v->counter), o, n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_cmpxchg_release(atomic##prefix##_t *v, \
|
|
c_t o, c_t n) \
|
|
{ \
|
|
return __cmpxchg_release(&(v->counter), o, n, size); \
|
|
} \
|
|
static __always_inline \
|
|
c_t atomic##prefix##_cmpxchg(atomic##prefix##_t *v, c_t o, c_t n) \
|
|
{ \
|
|
return __cmpxchg(&(v->counter), o, n, size); \
|
|
}
|
|
|
|
#ifdef CONFIG_GENERIC_ATOMIC64
|
|
#define ATOMIC_OPS() \
|
|
ATOMIC_OP( int, , 4)
|
|
#else
|
|
#define ATOMIC_OPS() \
|
|
ATOMIC_OP( int, , 4) \
|
|
ATOMIC_OP(long, 64, 8)
|
|
#endif
|
|
|
|
ATOMIC_OPS()
|
|
|
|
#undef ATOMIC_OPS
|
|
#undef ATOMIC_OP
|
|
|
|
static __always_inline int atomic_sub_if_positive(atomic_t *v, int offset)
|
|
{
|
|
int prev, rc;
|
|
|
|
__asm__ __volatile__ (
|
|
"0: lr.w %[p], %[c]\n"
|
|
" sub %[rc], %[p], %[o]\n"
|
|
" bltz %[rc], 1f\n"
|
|
" sc.w.rl %[rc], %[rc], %[c]\n"
|
|
" bnez %[rc], 0b\n"
|
|
" fence rw, rw\n"
|
|
"1:\n"
|
|
: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
|
|
: [o]"r" (offset)
|
|
: "memory");
|
|
return prev - offset;
|
|
}
|
|
|
|
#define atomic_dec_if_positive(v) atomic_sub_if_positive(v, 1)
|
|
|
|
#ifndef CONFIG_GENERIC_ATOMIC64
|
|
static __always_inline long atomic64_sub_if_positive(atomic64_t *v, int offset)
|
|
{
|
|
long prev, rc;
|
|
|
|
__asm__ __volatile__ (
|
|
"0: lr.d %[p], %[c]\n"
|
|
" sub %[rc], %[p], %[o]\n"
|
|
" bltz %[rc], 1f\n"
|
|
" sc.d.rl %[rc], %[rc], %[c]\n"
|
|
" bnez %[rc], 0b\n"
|
|
" fence rw, rw\n"
|
|
"1:\n"
|
|
: [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
|
|
: [o]"r" (offset)
|
|
: "memory");
|
|
return prev - offset;
|
|
}
|
|
|
|
#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(v, 1)
|
|
#endif
|
|
|
|
#endif /* _ASM_RISCV_ATOMIC_H */
|