mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 10:49:33 +00:00
![Davidlohr Bueso](/assets/img/avatar_default.png)
This is a nasty interface and setting the state of a foreign task must
not be done. As of the following commit:
be628be095
("bcache: Make gc wakeup sane, remove set_task_state()")
... everyone in the kernel calls set_task_state() with current, allowing
the helper to be removed.
However, as the comment indicates, it is still around for those archs
where computing current is more expensive than using a pointer, at least
in theory. An important arch that is affected is arm64, however this has
been addressed now [1] and performance is up to par making no difference
with either calls.
Of all the callers, if any, it's the locking bits that would care most
about this -- ie: we end up passing a tsk pointer to a lot of the lock
slowpath, and setting ->state on that. The following numbers are based
on two tests: a custom ad-hoc microbenchmark that just measures
latencies (for ~65 million calls) between get_task_state() vs
get_current_state().
Secondly for a higher overview, an unlink microbenchmark was used,
which pounds on a single file with open, close,unlink combos with
increasing thread counts (up to 4x ncpus). While the workload is quite
unrealistic, it does contend a lot on the inode mutex or now rwsem.
[1] https://lkml.kernel.org/r/1483468021-8237-1-git-send-email-mark.rutland@arm.com
== 1. x86-64 ==
Avg runtime set_task_state(): 601 msecs
Avg runtime set_current_state(): 552 msecs
vanilla dirty
Hmean unlink1-processes-2 36089.26 ( 0.00%) 38977.33 ( 8.00%)
Hmean unlink1-processes-5 28555.01 ( 0.00%) 29832.55 ( 4.28%)
Hmean unlink1-processes-8 37323.75 ( 0.00%) 44974.57 ( 20.50%)
Hmean unlink1-processes-12 43571.88 ( 0.00%) 44283.01 ( 1.63%)
Hmean unlink1-processes-21 34431.52 ( 0.00%) 38284.45 ( 11.19%)
Hmean unlink1-processes-30 34813.26 ( 0.00%) 37975.17 ( 9.08%)
Hmean unlink1-processes-48 37048.90 ( 0.00%) 39862.78 ( 7.59%)
Hmean unlink1-processes-79 35630.01 ( 0.00%) 36855.30 ( 3.44%)
Hmean unlink1-processes-110 36115.85 ( 0.00%) 39843.91 ( 10.32%)
Hmean unlink1-processes-141 32546.96 ( 0.00%) 35418.52 ( 8.82%)
Hmean unlink1-processes-172 34674.79 ( 0.00%) 36899.21 ( 6.42%)
Hmean unlink1-processes-203 37303.11 ( 0.00%) 36393.04 ( -2.44%)
Hmean unlink1-processes-224 35712.13 ( 0.00%) 36685.96 ( 2.73%)
== 2. ppc64le ==
Avg runtime set_task_state(): 938 msecs
Avg runtime set_current_state: 940 msecs
vanilla dirty
Hmean unlink1-processes-2 19269.19 ( 0.00%) 30704.50 ( 59.35%)
Hmean unlink1-processes-5 20106.15 ( 0.00%) 21804.15 ( 8.45%)
Hmean unlink1-processes-8 17496.97 ( 0.00%) 17243.28 ( -1.45%)
Hmean unlink1-processes-12 14224.15 ( 0.00%) 17240.21 ( 21.20%)
Hmean unlink1-processes-21 14155.66 ( 0.00%) 15681.23 ( 10.78%)
Hmean unlink1-processes-30 14450.70 ( 0.00%) 15995.83 ( 10.69%)
Hmean unlink1-processes-48 16945.57 ( 0.00%) 16370.42 ( -3.39%)
Hmean unlink1-processes-79 15788.39 ( 0.00%) 14639.27 ( -7.28%)
Hmean unlink1-processes-110 14268.48 ( 0.00%) 14377.40 ( 0.76%)
Hmean unlink1-processes-141 14023.65 ( 0.00%) 16271.69 ( 16.03%)
Hmean unlink1-processes-172 13417.62 ( 0.00%) 16067.55 ( 19.75%)
Hmean unlink1-processes-203 15293.08 ( 0.00%) 15440.40 ( 0.96%)
Hmean unlink1-processes-234 13719.32 ( 0.00%) 16190.74 ( 18.01%)
Hmean unlink1-processes-265 16400.97 ( 0.00%) 16115.22 ( -1.74%)
Hmean unlink1-processes-296 14388.60 ( 0.00%) 16216.13 ( 12.70%)
Hmean unlink1-processes-320 15771.85 ( 0.00%) 15905.96 ( 0.85%)
x86-64 (known to be fast for get_current()/this_cpu_read_stable() caching)
and ppc64 (with paca) show similar improvements in the unlink microbenches.
The small delta for ppc64 (2ms), does not represent the gains on the unlink
runs. In the case of x86, there was a decent amount of variation in the
latency runs, but always within a 20 to 50ms increase), ppc was more constant.
Signed-off-by: Davidlohr Bueso <dbueso@suse.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: dave@stgolabs.net
Cc: mark.rutland@arm.com
Link: http://lkml.kernel.org/r/1483479794-14013-5-git-send-email-dave@stgolabs.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>
455 lines
11 KiB
C
455 lines
11 KiB
C
/*
|
|
* Ldisc rw semaphore
|
|
*
|
|
* The ldisc semaphore is semantically a rw_semaphore but which enforces
|
|
* an alternate policy, namely:
|
|
* 1) Supports lock wait timeouts
|
|
* 2) Write waiter has priority
|
|
* 3) Downgrading is not supported
|
|
*
|
|
* Implementation notes:
|
|
* 1) Upper half of semaphore count is a wait count (differs from rwsem
|
|
* in that rwsem normalizes the upper half to the wait bias)
|
|
* 2) Lacks overflow checking
|
|
*
|
|
* The generic counting was copied and modified from include/asm-generic/rwsem.h
|
|
* by Paul Mackerras <paulus@samba.org>.
|
|
*
|
|
* The scheduling policy was copied and modified from lib/rwsem.c
|
|
* Written by David Howells (dhowells@redhat.com).
|
|
*
|
|
* This implementation incorporates the write lock stealing work of
|
|
* Michel Lespinasse <walken@google.com>.
|
|
*
|
|
* Copyright (C) 2013 Peter Hurley <peter@hurleysoftware.com>
|
|
*
|
|
* This file may be redistributed under the terms of the GNU General Public
|
|
* License v2.
|
|
*/
|
|
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/tty.h>
|
|
#include <linux/sched.h>
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
# define __acq(l, s, t, r, c, n, i) \
|
|
lock_acquire(&(l)->dep_map, s, t, r, c, n, i)
|
|
# define __rel(l, n, i) \
|
|
lock_release(&(l)->dep_map, n, i)
|
|
#define lockdep_acquire(l, s, t, i) __acq(l, s, t, 0, 1, NULL, i)
|
|
#define lockdep_acquire_nest(l, s, t, n, i) __acq(l, s, t, 0, 1, n, i)
|
|
#define lockdep_acquire_read(l, s, t, i) __acq(l, s, t, 1, 1, NULL, i)
|
|
#define lockdep_release(l, n, i) __rel(l, n, i)
|
|
#else
|
|
# define lockdep_acquire(l, s, t, i) do { } while (0)
|
|
# define lockdep_acquire_nest(l, s, t, n, i) do { } while (0)
|
|
# define lockdep_acquire_read(l, s, t, i) do { } while (0)
|
|
# define lockdep_release(l, n, i) do { } while (0)
|
|
#endif
|
|
|
|
#ifdef CONFIG_LOCK_STAT
|
|
# define lock_stat(_lock, stat) lock_##stat(&(_lock)->dep_map, _RET_IP_)
|
|
#else
|
|
# define lock_stat(_lock, stat) do { } while (0)
|
|
#endif
|
|
|
|
|
|
#if BITS_PER_LONG == 64
|
|
# define LDSEM_ACTIVE_MASK 0xffffffffL
|
|
#else
|
|
# define LDSEM_ACTIVE_MASK 0x0000ffffL
|
|
#endif
|
|
|
|
#define LDSEM_UNLOCKED 0L
|
|
#define LDSEM_ACTIVE_BIAS 1L
|
|
#define LDSEM_WAIT_BIAS (-LDSEM_ACTIVE_MASK-1)
|
|
#define LDSEM_READ_BIAS LDSEM_ACTIVE_BIAS
|
|
#define LDSEM_WRITE_BIAS (LDSEM_WAIT_BIAS + LDSEM_ACTIVE_BIAS)
|
|
|
|
struct ldsem_waiter {
|
|
struct list_head list;
|
|
struct task_struct *task;
|
|
};
|
|
|
|
static inline long ldsem_atomic_update(long delta, struct ld_semaphore *sem)
|
|
{
|
|
return atomic_long_add_return(delta, (atomic_long_t *)&sem->count);
|
|
}
|
|
|
|
/*
|
|
* ldsem_cmpxchg() updates @*old with the last-known sem->count value.
|
|
* Returns 1 if count was successfully changed; @*old will have @new value.
|
|
* Returns 0 if count was not changed; @*old will have most recent sem->count
|
|
*/
|
|
static inline int ldsem_cmpxchg(long *old, long new, struct ld_semaphore *sem)
|
|
{
|
|
long tmp = atomic_long_cmpxchg(&sem->count, *old, new);
|
|
if (tmp == *old) {
|
|
*old = new;
|
|
return 1;
|
|
} else {
|
|
*old = tmp;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Initialize an ldsem:
|
|
*/
|
|
void __init_ldsem(struct ld_semaphore *sem, const char *name,
|
|
struct lock_class_key *key)
|
|
{
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
/*
|
|
* Make sure we are not reinitializing a held semaphore:
|
|
*/
|
|
debug_check_no_locks_freed((void *)sem, sizeof(*sem));
|
|
lockdep_init_map(&sem->dep_map, name, key, 0);
|
|
#endif
|
|
sem->count = LDSEM_UNLOCKED;
|
|
sem->wait_readers = 0;
|
|
raw_spin_lock_init(&sem->wait_lock);
|
|
INIT_LIST_HEAD(&sem->read_wait);
|
|
INIT_LIST_HEAD(&sem->write_wait);
|
|
}
|
|
|
|
static void __ldsem_wake_readers(struct ld_semaphore *sem)
|
|
{
|
|
struct ldsem_waiter *waiter, *next;
|
|
struct task_struct *tsk;
|
|
long adjust, count;
|
|
|
|
/* Try to grant read locks to all readers on the read wait list.
|
|
* Note the 'active part' of the count is incremented by
|
|
* the number of readers before waking any processes up.
|
|
*/
|
|
adjust = sem->wait_readers * (LDSEM_ACTIVE_BIAS - LDSEM_WAIT_BIAS);
|
|
count = ldsem_atomic_update(adjust, sem);
|
|
do {
|
|
if (count > 0)
|
|
break;
|
|
if (ldsem_cmpxchg(&count, count - adjust, sem))
|
|
return;
|
|
} while (1);
|
|
|
|
list_for_each_entry_safe(waiter, next, &sem->read_wait, list) {
|
|
tsk = waiter->task;
|
|
smp_mb();
|
|
waiter->task = NULL;
|
|
wake_up_process(tsk);
|
|
put_task_struct(tsk);
|
|
}
|
|
INIT_LIST_HEAD(&sem->read_wait);
|
|
sem->wait_readers = 0;
|
|
}
|
|
|
|
static inline int writer_trylock(struct ld_semaphore *sem)
|
|
{
|
|
/* only wake this writer if the active part of the count can be
|
|
* transitioned from 0 -> 1
|
|
*/
|
|
long count = ldsem_atomic_update(LDSEM_ACTIVE_BIAS, sem);
|
|
do {
|
|
if ((count & LDSEM_ACTIVE_MASK) == LDSEM_ACTIVE_BIAS)
|
|
return 1;
|
|
if (ldsem_cmpxchg(&count, count - LDSEM_ACTIVE_BIAS, sem))
|
|
return 0;
|
|
} while (1);
|
|
}
|
|
|
|
static void __ldsem_wake_writer(struct ld_semaphore *sem)
|
|
{
|
|
struct ldsem_waiter *waiter;
|
|
|
|
waiter = list_entry(sem->write_wait.next, struct ldsem_waiter, list);
|
|
wake_up_process(waiter->task);
|
|
}
|
|
|
|
/*
|
|
* handle the lock release when processes blocked on it that can now run
|
|
* - if we come here from up_xxxx(), then:
|
|
* - the 'active part' of count (&0x0000ffff) reached 0 (but may have changed)
|
|
* - the 'waiting part' of count (&0xffff0000) is -ve (and will still be so)
|
|
* - the spinlock must be held by the caller
|
|
* - woken process blocks are discarded from the list after having task zeroed
|
|
*/
|
|
static void __ldsem_wake(struct ld_semaphore *sem)
|
|
{
|
|
if (!list_empty(&sem->write_wait))
|
|
__ldsem_wake_writer(sem);
|
|
else if (!list_empty(&sem->read_wait))
|
|
__ldsem_wake_readers(sem);
|
|
}
|
|
|
|
static void ldsem_wake(struct ld_semaphore *sem)
|
|
{
|
|
unsigned long flags;
|
|
|
|
raw_spin_lock_irqsave(&sem->wait_lock, flags);
|
|
__ldsem_wake(sem);
|
|
raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
|
|
}
|
|
|
|
/*
|
|
* wait for the read lock to be granted
|
|
*/
|
|
static struct ld_semaphore __sched *
|
|
down_read_failed(struct ld_semaphore *sem, long count, long timeout)
|
|
{
|
|
struct ldsem_waiter waiter;
|
|
long adjust = -LDSEM_ACTIVE_BIAS + LDSEM_WAIT_BIAS;
|
|
|
|
/* set up my own style of waitqueue */
|
|
raw_spin_lock_irq(&sem->wait_lock);
|
|
|
|
/* Try to reverse the lock attempt but if the count has changed
|
|
* so that reversing fails, check if there are are no waiters,
|
|
* and early-out if not */
|
|
do {
|
|
if (ldsem_cmpxchg(&count, count + adjust, sem))
|
|
break;
|
|
if (count > 0) {
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
return sem;
|
|
}
|
|
} while (1);
|
|
|
|
list_add_tail(&waiter.list, &sem->read_wait);
|
|
sem->wait_readers++;
|
|
|
|
waiter.task = current;
|
|
get_task_struct(current);
|
|
|
|
/* if there are no active locks, wake the new lock owner(s) */
|
|
if ((count & LDSEM_ACTIVE_MASK) == 0)
|
|
__ldsem_wake(sem);
|
|
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
|
|
/* wait to be given the lock */
|
|
for (;;) {
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
|
|
if (!waiter.task)
|
|
break;
|
|
if (!timeout)
|
|
break;
|
|
timeout = schedule_timeout(timeout);
|
|
}
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
if (!timeout) {
|
|
/* lock timed out but check if this task was just
|
|
* granted lock ownership - if so, pretend there
|
|
* was no timeout; otherwise, cleanup lock wait */
|
|
raw_spin_lock_irq(&sem->wait_lock);
|
|
if (waiter.task) {
|
|
ldsem_atomic_update(-LDSEM_WAIT_BIAS, sem);
|
|
list_del(&waiter.list);
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
put_task_struct(waiter.task);
|
|
return NULL;
|
|
}
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
}
|
|
|
|
return sem;
|
|
}
|
|
|
|
/*
|
|
* wait for the write lock to be granted
|
|
*/
|
|
static struct ld_semaphore __sched *
|
|
down_write_failed(struct ld_semaphore *sem, long count, long timeout)
|
|
{
|
|
struct ldsem_waiter waiter;
|
|
long adjust = -LDSEM_ACTIVE_BIAS;
|
|
int locked = 0;
|
|
|
|
/* set up my own style of waitqueue */
|
|
raw_spin_lock_irq(&sem->wait_lock);
|
|
|
|
/* Try to reverse the lock attempt but if the count has changed
|
|
* so that reversing fails, check if the lock is now owned,
|
|
* and early-out if so */
|
|
do {
|
|
if (ldsem_cmpxchg(&count, count + adjust, sem))
|
|
break;
|
|
if ((count & LDSEM_ACTIVE_MASK) == LDSEM_ACTIVE_BIAS) {
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
return sem;
|
|
}
|
|
} while (1);
|
|
|
|
list_add_tail(&waiter.list, &sem->write_wait);
|
|
|
|
waiter.task = current;
|
|
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
for (;;) {
|
|
if (!timeout)
|
|
break;
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
timeout = schedule_timeout(timeout);
|
|
raw_spin_lock_irq(&sem->wait_lock);
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
locked = writer_trylock(sem);
|
|
if (locked)
|
|
break;
|
|
}
|
|
|
|
if (!locked)
|
|
ldsem_atomic_update(-LDSEM_WAIT_BIAS, sem);
|
|
list_del(&waiter.list);
|
|
raw_spin_unlock_irq(&sem->wait_lock);
|
|
|
|
__set_current_state(TASK_RUNNING);
|
|
|
|
/* lock wait may have timed out */
|
|
if (!locked)
|
|
return NULL;
|
|
return sem;
|
|
}
|
|
|
|
|
|
|
|
static int __ldsem_down_read_nested(struct ld_semaphore *sem,
|
|
int subclass, long timeout)
|
|
{
|
|
long count;
|
|
|
|
lockdep_acquire_read(sem, subclass, 0, _RET_IP_);
|
|
|
|
count = ldsem_atomic_update(LDSEM_READ_BIAS, sem);
|
|
if (count <= 0) {
|
|
lock_stat(sem, contended);
|
|
if (!down_read_failed(sem, count, timeout)) {
|
|
lockdep_release(sem, 1, _RET_IP_);
|
|
return 0;
|
|
}
|
|
}
|
|
lock_stat(sem, acquired);
|
|
return 1;
|
|
}
|
|
|
|
static int __ldsem_down_write_nested(struct ld_semaphore *sem,
|
|
int subclass, long timeout)
|
|
{
|
|
long count;
|
|
|
|
lockdep_acquire(sem, subclass, 0, _RET_IP_);
|
|
|
|
count = ldsem_atomic_update(LDSEM_WRITE_BIAS, sem);
|
|
if ((count & LDSEM_ACTIVE_MASK) != LDSEM_ACTIVE_BIAS) {
|
|
lock_stat(sem, contended);
|
|
if (!down_write_failed(sem, count, timeout)) {
|
|
lockdep_release(sem, 1, _RET_IP_);
|
|
return 0;
|
|
}
|
|
}
|
|
lock_stat(sem, acquired);
|
|
return 1;
|
|
}
|
|
|
|
|
|
/*
|
|
* lock for reading -- returns 1 if successful, 0 if timed out
|
|
*/
|
|
int __sched ldsem_down_read(struct ld_semaphore *sem, long timeout)
|
|
{
|
|
might_sleep();
|
|
return __ldsem_down_read_nested(sem, 0, timeout);
|
|
}
|
|
|
|
/*
|
|
* trylock for reading -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int ldsem_down_read_trylock(struct ld_semaphore *sem)
|
|
{
|
|
long count = sem->count;
|
|
|
|
while (count >= 0) {
|
|
if (ldsem_cmpxchg(&count, count + LDSEM_READ_BIAS, sem)) {
|
|
lockdep_acquire_read(sem, 0, 1, _RET_IP_);
|
|
lock_stat(sem, acquired);
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* lock for writing -- returns 1 if successful, 0 if timed out
|
|
*/
|
|
int __sched ldsem_down_write(struct ld_semaphore *sem, long timeout)
|
|
{
|
|
might_sleep();
|
|
return __ldsem_down_write_nested(sem, 0, timeout);
|
|
}
|
|
|
|
/*
|
|
* trylock for writing -- returns 1 if successful, 0 if contention
|
|
*/
|
|
int ldsem_down_write_trylock(struct ld_semaphore *sem)
|
|
{
|
|
long count = sem->count;
|
|
|
|
while ((count & LDSEM_ACTIVE_MASK) == 0) {
|
|
if (ldsem_cmpxchg(&count, count + LDSEM_WRITE_BIAS, sem)) {
|
|
lockdep_acquire(sem, 0, 1, _RET_IP_);
|
|
lock_stat(sem, acquired);
|
|
return 1;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* release a read lock
|
|
*/
|
|
void ldsem_up_read(struct ld_semaphore *sem)
|
|
{
|
|
long count;
|
|
|
|
lockdep_release(sem, 1, _RET_IP_);
|
|
|
|
count = ldsem_atomic_update(-LDSEM_READ_BIAS, sem);
|
|
if (count < 0 && (count & LDSEM_ACTIVE_MASK) == 0)
|
|
ldsem_wake(sem);
|
|
}
|
|
|
|
/*
|
|
* release a write lock
|
|
*/
|
|
void ldsem_up_write(struct ld_semaphore *sem)
|
|
{
|
|
long count;
|
|
|
|
lockdep_release(sem, 1, _RET_IP_);
|
|
|
|
count = ldsem_atomic_update(-LDSEM_WRITE_BIAS, sem);
|
|
if (count < 0)
|
|
ldsem_wake(sem);
|
|
}
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
int ldsem_down_read_nested(struct ld_semaphore *sem, int subclass, long timeout)
|
|
{
|
|
might_sleep();
|
|
return __ldsem_down_read_nested(sem, subclass, timeout);
|
|
}
|
|
|
|
int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass,
|
|
long timeout)
|
|
{
|
|
might_sleep();
|
|
return __ldsem_down_write_nested(sem, subclass, timeout);
|
|
}
|
|
|
|
#endif
|