mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 19:29:37 +00:00
![Peter Zijlstra](/assets/img/avatar_default.png)
Currently instrumentation of atomic primitives is done at the architecture level, while composites or fallbacks are provided at the generic level. The result is that there are no uninstrumented variants of the fallbacks. Since there is now need of such variants to isolate text poke from any form of instrumentation invert this ordering. Doing this means moving the instrumentation into the generic code as well as having (for now) two variants of the fallbacks. Notes: - the various *cond_read* primitives are not proper fallbacks and got moved into linux/atomic.c. No arch_ variants are generated because the base primitives smp_cond_load*() are instrumented. - once all architectures are moved over to arch_atomic_ one of the fallback variants can be removed and some 2300 lines reclaimed. - atomic_{read,set}*() are no longer double-instrumented Reported-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Mark Rutland <mark.rutland@arm.com> Link: https://lkml.kernel.org/r/20200505134058.769149955@linutronix.de
89 lines
2.6 KiB
C
89 lines
2.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/* Atomic operations usable in machine independent code */
|
|
#ifndef _LINUX_ATOMIC_H
|
|
#define _LINUX_ATOMIC_H
|
|
#include <linux/types.h>
|
|
|
|
#include <asm/atomic.h>
|
|
#include <asm/barrier.h>
|
|
|
|
/*
|
|
* Relaxed variants of xchg, cmpxchg and some atomic operations.
|
|
*
|
|
* We support four variants:
|
|
*
|
|
* - Fully ordered: The default implementation, no suffix required.
|
|
* - Acquire: Provides ACQUIRE semantics, _acquire suffix.
|
|
* - Release: Provides RELEASE semantics, _release suffix.
|
|
* - Relaxed: No ordering guarantees, _relaxed suffix.
|
|
*
|
|
* For compound atomics performing both a load and a store, ACQUIRE
|
|
* semantics apply only to the load and RELEASE semantics only to the
|
|
* store portion of the operation. Note that a failed cmpxchg_acquire
|
|
* does -not- imply any memory ordering constraints.
|
|
*
|
|
* See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
|
|
*/
|
|
|
|
#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
|
|
#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
|
|
|
|
#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
|
|
#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
|
|
|
|
/*
|
|
* The idea here is to build acquire/release variants by adding explicit
|
|
* barriers on top of the relaxed variant. In the case where the relaxed
|
|
* variant is already fully ordered, no additional barriers are needed.
|
|
*
|
|
* If an architecture overrides __atomic_acquire_fence() it will probably
|
|
* want to define smp_mb__after_spinlock().
|
|
*/
|
|
#ifndef __atomic_acquire_fence
|
|
#define __atomic_acquire_fence smp_mb__after_atomic
|
|
#endif
|
|
|
|
#ifndef __atomic_release_fence
|
|
#define __atomic_release_fence smp_mb__before_atomic
|
|
#endif
|
|
|
|
#ifndef __atomic_pre_full_fence
|
|
#define __atomic_pre_full_fence smp_mb__before_atomic
|
|
#endif
|
|
|
|
#ifndef __atomic_post_full_fence
|
|
#define __atomic_post_full_fence smp_mb__after_atomic
|
|
#endif
|
|
|
|
#define __atomic_op_acquire(op, args...) \
|
|
({ \
|
|
typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
|
|
__atomic_acquire_fence(); \
|
|
__ret; \
|
|
})
|
|
|
|
#define __atomic_op_release(op, args...) \
|
|
({ \
|
|
__atomic_release_fence(); \
|
|
op##_relaxed(args); \
|
|
})
|
|
|
|
#define __atomic_op_fence(op, args...) \
|
|
({ \
|
|
typeof(op##_relaxed(args)) __ret; \
|
|
__atomic_pre_full_fence(); \
|
|
__ret = op##_relaxed(args); \
|
|
__atomic_post_full_fence(); \
|
|
__ret; \
|
|
})
|
|
|
|
#ifdef ARCH_ATOMIC
|
|
#include <linux/atomic-arch-fallback.h>
|
|
#include <asm-generic/atomic-instrumented.h>
|
|
#else
|
|
#include <linux/atomic-fallback.h>
|
|
#endif
|
|
|
|
#include <asm-generic/atomic-long.h>
|
|
|
|
#endif /* _LINUX_ATOMIC_H */
|