mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-30 14:19:16 +00:00
futex,rt_mutex: Provide futex specific rt_mutex API
Part of what makes futex_unlock_pi() intricate is that rt_mutex_futex_unlock() -> rt_mutex_slowunlock() can drop rt_mutex::wait_lock. This means it cannot rely on the atomicy of wait_lock, which would be preferred in order to not rely on hb->lock so much. The reason rt_mutex_slowunlock() needs to drop wait_lock is because it can race with the rt_mutex fastpath, however futexes have their own fast path. Since futexes already have a bunch of separate rt_mutex accessors, complete that set and implement a rt_mutex variant without fastpath for them. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: juri.lelli@arm.com Cc: bigeasy@linutronix.de Cc: xlpang@redhat.com Cc: rostedt@goodmis.org Cc: mathieu.desnoyers@efficios.com Cc: jdesfossez@efficios.com Cc: dvhart@infradead.org Cc: bristot@redhat.com Link: http://lkml.kernel.org/r/20170322104151.702962446@infradead.org Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
This commit is contained in:
parent
fffa954fb5
commit
5293c2efda
3 changed files with 62 additions and 32 deletions
|
@ -916,7 +916,7 @@ void exit_pi_state_list(struct task_struct *curr)
|
||||||
pi_state->owner = NULL;
|
pi_state->owner = NULL;
|
||||||
raw_spin_unlock_irq(&curr->pi_lock);
|
raw_spin_unlock_irq(&curr->pi_lock);
|
||||||
|
|
||||||
rt_mutex_unlock(&pi_state->pi_mutex);
|
rt_mutex_futex_unlock(&pi_state->pi_mutex);
|
||||||
|
|
||||||
spin_unlock(&hb->lock);
|
spin_unlock(&hb->lock);
|
||||||
|
|
||||||
|
@ -1364,20 +1364,18 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter
|
||||||
pi_state->owner = new_owner;
|
pi_state->owner = new_owner;
|
||||||
raw_spin_unlock(&new_owner->pi_lock);
|
raw_spin_unlock(&new_owner->pi_lock);
|
||||||
|
|
||||||
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
|
|
||||||
|
|
||||||
deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* First unlock HB so the waiter does not spin on it once he got woken
|
* We've updated the uservalue, this unlock cannot fail.
|
||||||
* up. Second wake up the waiter before the priority is adjusted. If we
|
|
||||||
* deboost first (and lose our higher priority), then the task might get
|
|
||||||
* scheduled away before the wake up can take place.
|
|
||||||
*/
|
*/
|
||||||
|
deboost = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
|
||||||
|
|
||||||
|
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
|
||||||
spin_unlock(&hb->lock);
|
spin_unlock(&hb->lock);
|
||||||
wake_up_q(&wake_q);
|
|
||||||
if (deboost)
|
if (deboost) {
|
||||||
|
wake_up_q(&wake_q);
|
||||||
rt_mutex_adjust_prio(current);
|
rt_mutex_adjust_prio(current);
|
||||||
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
@ -2253,7 +2251,7 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
|
||||||
* task acquired the rt_mutex after we removed ourself from the
|
* task acquired the rt_mutex after we removed ourself from the
|
||||||
* rt_mutex waiters list.
|
* rt_mutex waiters list.
|
||||||
*/
|
*/
|
||||||
if (rt_mutex_trylock(&q->pi_state->pi_mutex)) {
|
if (rt_mutex_futex_trylock(&q->pi_state->pi_mutex)) {
|
||||||
locked = 1;
|
locked = 1;
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
|
@ -2568,7 +2566,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
|
||||||
if (!trylock) {
|
if (!trylock) {
|
||||||
ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
|
ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
|
||||||
} else {
|
} else {
|
||||||
ret = rt_mutex_trylock(&q.pi_state->pi_mutex);
|
ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
|
||||||
/* Fixup the trylock return value: */
|
/* Fixup the trylock return value: */
|
||||||
ret = ret ? 0 : -EWOULDBLOCK;
|
ret = ret ? 0 : -EWOULDBLOCK;
|
||||||
}
|
}
|
||||||
|
@ -2591,7 +2589,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
|
||||||
* it and return the fault to userspace.
|
* it and return the fault to userspace.
|
||||||
*/
|
*/
|
||||||
if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))
|
if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))
|
||||||
rt_mutex_unlock(&q.pi_state->pi_mutex);
|
rt_mutex_futex_unlock(&q.pi_state->pi_mutex);
|
||||||
|
|
||||||
/* Unqueue and drop the lock */
|
/* Unqueue and drop the lock */
|
||||||
unqueue_me_pi(&q);
|
unqueue_me_pi(&q);
|
||||||
|
@ -2898,7 +2896,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
|
||||||
spin_lock(q.lock_ptr);
|
spin_lock(q.lock_ptr);
|
||||||
ret = fixup_pi_state_owner(uaddr2, &q, current);
|
ret = fixup_pi_state_owner(uaddr2, &q, current);
|
||||||
if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current)
|
if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current)
|
||||||
rt_mutex_unlock(&q.pi_state->pi_mutex);
|
rt_mutex_futex_unlock(&q.pi_state->pi_mutex);
|
||||||
/*
|
/*
|
||||||
* Drop the reference to the pi state which
|
* Drop the reference to the pi state which
|
||||||
* the requeue_pi() code acquired for us.
|
* the requeue_pi() code acquired for us.
|
||||||
|
@ -2938,7 +2936,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
|
||||||
* userspace.
|
* userspace.
|
||||||
*/
|
*/
|
||||||
if (ret && rt_mutex_owner(pi_mutex) == current)
|
if (ret && rt_mutex_owner(pi_mutex) == current)
|
||||||
rt_mutex_unlock(pi_mutex);
|
rt_mutex_futex_unlock(pi_mutex);
|
||||||
|
|
||||||
/* Unqueue and drop the lock. */
|
/* Unqueue and drop the lock. */
|
||||||
unqueue_me_pi(&q);
|
unqueue_me_pi(&q);
|
||||||
|
|
|
@ -1488,15 +1488,23 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Futex variant with full deadlock detection.
|
* Futex variant with full deadlock detection.
|
||||||
|
* Futex variants must not use the fast-path, see __rt_mutex_futex_unlock().
|
||||||
*/
|
*/
|
||||||
int rt_mutex_timed_futex_lock(struct rt_mutex *lock,
|
int __sched rt_mutex_timed_futex_lock(struct rt_mutex *lock,
|
||||||
struct hrtimer_sleeper *timeout)
|
struct hrtimer_sleeper *timeout)
|
||||||
{
|
{
|
||||||
might_sleep();
|
might_sleep();
|
||||||
|
|
||||||
return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
|
return rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE,
|
||||||
RT_MUTEX_FULL_CHAINWALK,
|
timeout, RT_MUTEX_FULL_CHAINWALK);
|
||||||
rt_mutex_slowlock);
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Futex variant, must not use fastpath.
|
||||||
|
*/
|
||||||
|
int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
|
||||||
|
{
|
||||||
|
return rt_mutex_slowtrylock(lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1555,19 +1563,38 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock)
|
||||||
EXPORT_SYMBOL_GPL(rt_mutex_unlock);
|
EXPORT_SYMBOL_GPL(rt_mutex_unlock);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock
|
* Futex variant, that since futex variants do not use the fast-path, can be
|
||||||
* @lock: the rt_mutex to be unlocked
|
* simple and will not need to retry.
|
||||||
*
|
|
||||||
* Returns: true/false indicating whether priority adjustment is
|
|
||||||
* required or not.
|
|
||||||
*/
|
*/
|
||||||
bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock,
|
bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
|
||||||
struct wake_q_head *wqh)
|
struct wake_q_head *wake_q)
|
||||||
{
|
{
|
||||||
if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
|
lockdep_assert_held(&lock->wait_lock);
|
||||||
return false;
|
|
||||||
|
|
||||||
return rt_mutex_slowunlock(lock, wqh);
|
debug_rt_mutex_unlock(lock);
|
||||||
|
|
||||||
|
if (!rt_mutex_has_waiters(lock)) {
|
||||||
|
lock->owner = NULL;
|
||||||
|
return false; /* done */
|
||||||
|
}
|
||||||
|
|
||||||
|
mark_wakeup_next_waiter(wake_q, lock);
|
||||||
|
return true; /* deboost and wakeups */
|
||||||
|
}
|
||||||
|
|
||||||
|
void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
|
||||||
|
{
|
||||||
|
DEFINE_WAKE_Q(wake_q);
|
||||||
|
bool deboost;
|
||||||
|
|
||||||
|
raw_spin_lock_irq(&lock->wait_lock);
|
||||||
|
deboost = __rt_mutex_futex_unlock(lock, &wake_q);
|
||||||
|
raw_spin_unlock_irq(&lock->wait_lock);
|
||||||
|
|
||||||
|
if (deboost) {
|
||||||
|
wake_up_q(&wake_q);
|
||||||
|
rt_mutex_adjust_prio(current);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -109,9 +109,14 @@ extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
|
||||||
extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
|
extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
|
||||||
struct hrtimer_sleeper *to,
|
struct hrtimer_sleeper *to,
|
||||||
struct rt_mutex_waiter *waiter);
|
struct rt_mutex_waiter *waiter);
|
||||||
|
|
||||||
extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
|
extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
|
||||||
extern bool rt_mutex_futex_unlock(struct rt_mutex *lock,
|
extern int rt_mutex_futex_trylock(struct rt_mutex *l);
|
||||||
struct wake_q_head *wqh);
|
|
||||||
|
extern void rt_mutex_futex_unlock(struct rt_mutex *lock);
|
||||||
|
extern bool __rt_mutex_futex_unlock(struct rt_mutex *lock,
|
||||||
|
struct wake_q_head *wqh);
|
||||||
|
|
||||||
extern void rt_mutex_adjust_prio(struct task_struct *task);
|
extern void rt_mutex_adjust_prio(struct task_struct *task);
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_RT_MUTEXES
|
#ifdef CONFIG_DEBUG_RT_MUTEXES
|
||||||
|
|
Loading…
Reference in a new issue