mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-30 08:02:30 +00:00
52b94129f2
The use of any kind of wait queue is an overkill for pcpu-rwsems. While one option would be to use the less heavy simple (swait) flavor, this is still too much for what pcpu-rwsems needs. For one, we do not care about any sort of queuing in that the only (rare) time writers (and readers, for that matter) are queued is when trying to acquire the regular contended rw_sem. There cannot be any further queuing as writers are serialized by the rw_sem in the first place. Given that percpu_down_write() must not be called after exit_notify(), we can replace the bulky waitqueue with rcuwait such that a writer can wait for its turn to take the lock. As such, we can avoid the queue handling and locking overhead. Signed-off-by: Davidlohr Bueso <dbueso@suse.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Oleg Nesterov <oleg@redhat.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: dave@stgolabs.net Link: http://lkml.kernel.org/r/1484148146-14210-3-git-send-email-dave@stgolabs.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
145 lines
4 KiB
C
145 lines
4 KiB
C
#ifndef _LINUX_PERCPU_RWSEM_H
|
|
#define _LINUX_PERCPU_RWSEM_H
|
|
|
|
#include <linux/atomic.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/rcuwait.h>
|
|
#include <linux/rcu_sync.h>
|
|
#include <linux/lockdep.h>
|
|
|
|
struct percpu_rw_semaphore {
|
|
struct rcu_sync rss;
|
|
unsigned int __percpu *read_count;
|
|
struct rw_semaphore rw_sem; /* slowpath */
|
|
struct rcuwait writer; /* blocked writer */
|
|
int readers_block;
|
|
};
|
|
|
|
#define DEFINE_STATIC_PERCPU_RWSEM(name) \
|
|
static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \
|
|
static struct percpu_rw_semaphore name = { \
|
|
.rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC), \
|
|
.read_count = &__percpu_rwsem_rc_##name, \
|
|
.rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \
|
|
.writer = __RCUWAIT_INITIALIZER(name.writer), \
|
|
}
|
|
|
|
extern int __percpu_down_read(struct percpu_rw_semaphore *, int);
|
|
extern void __percpu_up_read(struct percpu_rw_semaphore *);
|
|
|
|
static inline void percpu_down_read_preempt_disable(struct percpu_rw_semaphore *sem)
|
|
{
|
|
might_sleep();
|
|
|
|
rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_);
|
|
|
|
preempt_disable();
|
|
/*
|
|
* We are in an RCU-sched read-side critical section, so the writer
|
|
* cannot both change sem->state from readers_fast and start checking
|
|
* counters while we are here. So if we see !sem->state, we know that
|
|
* the writer won't be checking until we're past the preempt_enable()
|
|
* and that one the synchronize_sched() is done, the writer will see
|
|
* anything we did within this RCU-sched read-size critical section.
|
|
*/
|
|
__this_cpu_inc(*sem->read_count);
|
|
if (unlikely(!rcu_sync_is_idle(&sem->rss)))
|
|
__percpu_down_read(sem, false); /* Unconditional memory barrier */
|
|
barrier();
|
|
/*
|
|
* The barrier() prevents the compiler from
|
|
* bleeding the critical section out.
|
|
*/
|
|
}
|
|
|
|
static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
|
|
{
|
|
percpu_down_read_preempt_disable(sem);
|
|
preempt_enable();
|
|
}
|
|
|
|
static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
|
|
{
|
|
int ret = 1;
|
|
|
|
preempt_disable();
|
|
/*
|
|
* Same as in percpu_down_read().
|
|
*/
|
|
__this_cpu_inc(*sem->read_count);
|
|
if (unlikely(!rcu_sync_is_idle(&sem->rss)))
|
|
ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */
|
|
preempt_enable();
|
|
/*
|
|
* The barrier() from preempt_enable() prevents the compiler from
|
|
* bleeding the critical section out.
|
|
*/
|
|
|
|
if (ret)
|
|
rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static inline void percpu_up_read_preempt_enable(struct percpu_rw_semaphore *sem)
|
|
{
|
|
/*
|
|
* The barrier() prevents the compiler from
|
|
* bleeding the critical section out.
|
|
*/
|
|
barrier();
|
|
/*
|
|
* Same as in percpu_down_read().
|
|
*/
|
|
if (likely(rcu_sync_is_idle(&sem->rss)))
|
|
__this_cpu_dec(*sem->read_count);
|
|
else
|
|
__percpu_up_read(sem); /* Unconditional memory barrier */
|
|
preempt_enable();
|
|
|
|
rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_);
|
|
}
|
|
|
|
static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
|
|
{
|
|
preempt_disable();
|
|
percpu_up_read_preempt_enable(sem);
|
|
}
|
|
|
|
extern void percpu_down_write(struct percpu_rw_semaphore *);
|
|
extern void percpu_up_write(struct percpu_rw_semaphore *);
|
|
|
|
extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
|
|
const char *, struct lock_class_key *);
|
|
|
|
extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
|
|
|
|
#define percpu_init_rwsem(sem) \
|
|
({ \
|
|
static struct lock_class_key rwsem_key; \
|
|
__percpu_init_rwsem(sem, #sem, &rwsem_key); \
|
|
})
|
|
|
|
#define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem)
|
|
|
|
#define percpu_rwsem_assert_held(sem) \
|
|
lockdep_assert_held(&(sem)->rw_sem)
|
|
|
|
static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
|
|
bool read, unsigned long ip)
|
|
{
|
|
lock_release(&sem->rw_sem.dep_map, 1, ip);
|
|
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
|
|
if (!read)
|
|
sem->rw_sem.owner = NULL;
|
|
#endif
|
|
}
|
|
|
|
static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
|
|
bool read, unsigned long ip)
|
|
{
|
|
lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);
|
|
}
|
|
|
|
#endif
|