mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 19:00:25 +00:00
![Waiman Long](/assets/img/avatar_default.png)
For an uncontended rwsem, count and owner are the only fields a task needs to touch when acquiring the rwsem. So they are put next to each other to increase the chance that they will share the same cacheline. On a ThunderX2 99xx (arm64) system with 32K L1 cache and 256K L2 cache, a rwsem locking microbenchmark with one locking thread was run to write-lock and write-unlock an array of rwsems separated 2 cachelines apart in a 1M byte memory block. The locking rates (kops/s) of the microbenchmark when the rwsems are at various "long" (8-byte) offsets from beginning of the cacheline before and after the patch were as follows: Cacheline Offset Pre-patch Post-patch ---------------- --------- ---------- 0 17,449 16,588 1 17,450 16,465 2 17,450 16,460 3 17,453 16,462 4 14,867 16,471 5 14,867 16,470 6 14,853 16,464 7 14,867 13,172 Before the patch, the count and owner are 4 "long"s apart. After the patch, they are only 1 "long" apart. The rwsem data have to be loaded from the L3 cache for each access. It can be seen that the locking rates are more consistent after the patch than before. Note that for this particular system, the performance drop happens whenever the count and owner are at an odd multiples of "long"s apart. No performance drop was observed when only a single rwsem was used (hot cache). So the drop is likely just an idiosyncrasy of the cache architecture of this chip than an inherent problem with the patch. Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Waiman Long <longman@redhat.com> Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Borislav Petkov <bp@alien8.de> Cc: Davidlohr Bueso <dave@stgolabs.net> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Tim Chen <tim.c.chen@linux.intel.com> Cc: Will Deacon <will.deacon@arm.com> Link: http://lkml.kernel.org/r/20190404174320.22416-12-longman@redhat.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
191 lines
5.8 KiB
C
191 lines
5.8 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/* rwsem.h: R/W semaphores, public interface
|
|
*
|
|
* Written by David Howells (dhowells@redhat.com).
|
|
* Derived from asm-i386/semaphore.h
|
|
*/
|
|
|
|
#ifndef _LINUX_RWSEM_H
|
|
#define _LINUX_RWSEM_H
|
|
|
|
#include <linux/linkage.h>
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/err.h>
|
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
|
#include <linux/osq_lock.h>
|
|
#endif
|
|
|
|
/*
|
|
* For an uncontended rwsem, count and owner are the only fields a task
|
|
* needs to touch when acquiring the rwsem. So they are put next to each
|
|
* other to increase the chance that they will share the same cacheline.
|
|
*
|
|
* In a contended rwsem, the owner is likely the most frequently accessed
|
|
* field in the structure as the optimistic waiter that holds the osq lock
|
|
* will spin on owner. For an embedded rwsem, other hot fields in the
|
|
* containing structure should be moved further away from the rwsem to
|
|
* reduce the chance that they will share the same cacheline causing
|
|
* cacheline bouncing problem.
|
|
*/
|
|
struct rw_semaphore {
|
|
atomic_long_t count;
|
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
|
/*
|
|
* Write owner. Used as a speculative check to see
|
|
* if the owner is running on the cpu.
|
|
*/
|
|
struct task_struct *owner;
|
|
struct optimistic_spin_queue osq; /* spinner MCS lock */
|
|
#endif
|
|
raw_spinlock_t wait_lock;
|
|
struct list_head wait_list;
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
struct lockdep_map dep_map;
|
|
#endif
|
|
};
|
|
|
|
/*
|
|
* Setting bit 1 of the owner field but not bit 0 will indicate
|
|
* that the rwsem is writer-owned with an unknown owner.
|
|
*/
|
|
#define RWSEM_OWNER_UNKNOWN ((struct task_struct *)-2L)
|
|
|
|
/* In all implementations count != 0 means locked */
|
|
static inline int rwsem_is_locked(struct rw_semaphore *sem)
|
|
{
|
|
return atomic_long_read(&sem->count) != 0;
|
|
}
|
|
|
|
#define RWSEM_UNLOCKED_VALUE 0L
|
|
#define __RWSEM_INIT_COUNT(name) .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)
|
|
|
|
/* Common initializer macros and functions */
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname }
|
|
#else
|
|
# define __RWSEM_DEP_MAP_INIT(lockname)
|
|
#endif
|
|
|
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
|
#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED, .owner = NULL
|
|
#else
|
|
#define __RWSEM_OPT_INIT(lockname)
|
|
#endif
|
|
|
|
#define __RWSEM_INITIALIZER(name) \
|
|
{ __RWSEM_INIT_COUNT(name), \
|
|
.wait_list = LIST_HEAD_INIT((name).wait_list), \
|
|
.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \
|
|
__RWSEM_OPT_INIT(name) \
|
|
__RWSEM_DEP_MAP_INIT(name) }
|
|
|
|
#define DECLARE_RWSEM(name) \
|
|
struct rw_semaphore name = __RWSEM_INITIALIZER(name)
|
|
|
|
extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
|
|
struct lock_class_key *key);
|
|
|
|
#define init_rwsem(sem) \
|
|
do { \
|
|
static struct lock_class_key __key; \
|
|
\
|
|
__init_rwsem((sem), #sem, &__key); \
|
|
} while (0)
|
|
|
|
/*
|
|
* This is the same regardless of which rwsem implementation that is being used.
|
|
* It is just a heuristic meant to be called by somebody alreadying holding the
|
|
* rwsem to see if somebody from an incompatible type is wanting access to the
|
|
* lock.
|
|
*/
|
|
static inline int rwsem_is_contended(struct rw_semaphore *sem)
|
|
{
|
|
return !list_empty(&sem->wait_list);
|
|
}
|
|
|
|
/*
|
|
* lock for reading
|
|
*/
|
|
extern void down_read(struct rw_semaphore *sem);
|
|
extern int __must_check down_read_killable(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* trylock for reading -- returns 1 if successful, 0 if contention
|
|
*/
|
|
extern int down_read_trylock(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* lock for writing
|
|
*/
|
|
extern void down_write(struct rw_semaphore *sem);
|
|
extern int __must_check down_write_killable(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* trylock for writing -- returns 1 if successful, 0 if contention
|
|
*/
|
|
extern int down_write_trylock(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* release a read lock
|
|
*/
|
|
extern void up_read(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* release a write lock
|
|
*/
|
|
extern void up_write(struct rw_semaphore *sem);
|
|
|
|
/*
|
|
* downgrade write lock to read lock
|
|
*/
|
|
extern void downgrade_write(struct rw_semaphore *sem);
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
/*
|
|
* nested locking. NOTE: rwsems are not allowed to recurse
|
|
* (which occurs if the same task tries to acquire the same
|
|
* lock instance multiple times), but multiple locks of the
|
|
* same lock class might be taken, if the order of the locks
|
|
* is always the same. This ordering rule can be expressed
|
|
* to lockdep via the _nested() APIs, but enumerating the
|
|
* subclasses that are used. (If the nesting relationship is
|
|
* static then another method for expressing nested locking is
|
|
* the explicit definition of lock class keys and the use of
|
|
* lockdep_set_class() at lock initialization time.
|
|
* See Documentation/locking/lockdep-design.txt for more details.)
|
|
*/
|
|
extern void down_read_nested(struct rw_semaphore *sem, int subclass);
|
|
extern void down_write_nested(struct rw_semaphore *sem, int subclass);
|
|
extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass);
|
|
extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock);
|
|
|
|
# define down_write_nest_lock(sem, nest_lock) \
|
|
do { \
|
|
typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
|
|
_down_write_nest_lock(sem, &(nest_lock)->dep_map); \
|
|
} while (0);
|
|
|
|
/*
|
|
* Take/release a lock when not the owner will release it.
|
|
*
|
|
* [ This API should be avoided as much as possible - the
|
|
* proper abstraction for this case is completions. ]
|
|
*/
|
|
extern void down_read_non_owner(struct rw_semaphore *sem);
|
|
extern void up_read_non_owner(struct rw_semaphore *sem);
|
|
#else
|
|
# define down_read_nested(sem, subclass) down_read(sem)
|
|
# define down_write_nest_lock(sem, nest_lock) down_write(sem)
|
|
# define down_write_nested(sem, subclass) down_write(sem)
|
|
# define down_write_killable_nested(sem, subclass) down_write_killable(sem)
|
|
# define down_read_non_owner(sem) down_read(sem)
|
|
# define up_read_non_owner(sem) up_read(sem)
|
|
#endif
|
|
|
|
#endif /* _LINUX_RWSEM_H */
|