mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-05 02:19:51 +00:00
![Paul E. McKenney](/assets/img/avatar_default.png)
Inspection is proving insufficient to catch all RCU misuses, which is understandable given that rcu_dereference() might be protected by any of four different flavors of RCU (RCU, RCU-bh, RCU-sched, and SRCU), and might also/instead be protected by any of a number of locking primitives. It is therefore time to enlist the aid of lockdep. This set of patches is inspired by earlier work by Peter Zijlstra and Thomas Gleixner, and takes the following approach: o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched. o Set up separate lockdep classes for each instance of SRCU. o Create primitives that check for being in an RCU read-side critical section. These return exact answers if lockdep is fully enabled, but if unsure, report being in an RCU read-side critical section. (We want to avoid false positives!) The primitives are: For RCU: rcu_read_lock_held(void) For RCU-bh: rcu_read_lock_bh_held(void) For RCU-sched: rcu_read_lock_sched_held(void) For SRCU: srcu_read_lock_held(struct srcu_struct *sp) o Add rcu_dereference_check(), which takes a second argument in which one places a boolean expression based on the above primitives and/or lockdep_is_held(). o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING, and should be quite helpful during the transition period while CONFIG_PROVE_RCU-unaware patches are in flight. The existing rcu_dereference() primitive does no checking, but upcoming patches will change that. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: mathieu.desnoyers@polymtl.ca Cc: josh@joshtriplett.org Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org Cc: Valdis.Kletnieks@vt.edu Cc: dhowells@redhat.com LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
408 lines
13 KiB
C
408 lines
13 KiB
C
/*
|
|
* Read-Copy Update mechanism for mutual exclusion
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, write to the Free Software
|
|
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
*
|
|
* Copyright IBM Corporation, 2001
|
|
*
|
|
* Author: Dipankar Sarma <dipankar@in.ibm.com>
|
|
*
|
|
* Based on the original work by Paul McKenney <paulmck@us.ibm.com>
|
|
* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
|
|
* Papers:
|
|
* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
|
|
* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
|
|
*
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
|
* http://lse.sourceforge.net/locking/rcupdate.html
|
|
*
|
|
*/
|
|
|
|
#ifndef __LINUX_RCUPDATE_H
|
|
#define __LINUX_RCUPDATE_H
|
|
|
|
#include <linux/cache.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/seqlock.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/completion.h>
|
|
|
|
/**
|
|
* struct rcu_head - callback structure for use with RCU
|
|
* @next: next update requests in a list
|
|
* @func: actual update function to call after the grace period.
|
|
*/
|
|
struct rcu_head {
|
|
struct rcu_head *next;
|
|
void (*func)(struct rcu_head *head);
|
|
};
|
|
|
|
/* Exported common interfaces */
|
|
extern void synchronize_rcu_bh(void);
|
|
extern void synchronize_sched(void);
|
|
extern void rcu_barrier(void);
|
|
extern void rcu_barrier_bh(void);
|
|
extern void rcu_barrier_sched(void);
|
|
extern void synchronize_sched_expedited(void);
|
|
extern int sched_expedited_torture_stats(char *page);
|
|
|
|
/* Internal to kernel */
|
|
extern void rcu_init(void);
|
|
|
|
#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
|
|
#include <linux/rcutree.h>
|
|
#elif defined(CONFIG_TINY_RCU)
|
|
#include <linux/rcutiny.h>
|
|
#else
|
|
#error "Unknown RCU implementation specified to kernel configuration"
|
|
#endif
|
|
|
|
#define RCU_HEAD_INIT { .next = NULL, .func = NULL }
|
|
#define RCU_HEAD(head) struct rcu_head head = RCU_HEAD_INIT
|
|
#define INIT_RCU_HEAD(ptr) do { \
|
|
(ptr)->next = NULL; (ptr)->func = NULL; \
|
|
} while (0)
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
extern struct lockdep_map rcu_lock_map;
|
|
# define rcu_read_acquire() \
|
|
lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
|
|
|
|
extern struct lockdep_map rcu_bh_lock_map;
|
|
# define rcu_read_acquire_bh() \
|
|
lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
# define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_)
|
|
|
|
extern struct lockdep_map rcu_sched_lock_map;
|
|
# define rcu_read_acquire_sched() \
|
|
lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
|
|
# define rcu_read_release_sched() \
|
|
lock_release(&rcu_sched_lock_map, 1, _THIS_IP_)
|
|
|
|
/**
|
|
* rcu_read_lock_held - might we be in RCU read-side critical section?
|
|
*
|
|
* If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in
|
|
* an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
* this assumes we are in an RCU read-side critical section unless it can
|
|
* prove otherwise.
|
|
*/
|
|
static inline int rcu_read_lock_held(void)
|
|
{
|
|
if (debug_locks)
|
|
return lock_is_held(&rcu_lock_map);
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section?
|
|
*
|
|
* If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in
|
|
* an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
* this assumes we are in an RCU-bh read-side critical section unless it can
|
|
* prove otherwise.
|
|
*/
|
|
static inline int rcu_read_lock_bh_held(void)
|
|
{
|
|
if (debug_locks)
|
|
return lock_is_held(&rcu_bh_lock_map);
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section?
|
|
*
|
|
* If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in an
|
|
* RCU-sched read-side critical section. In absence of CONFIG_PROVE_LOCKING,
|
|
* this assumes we are in an RCU-sched read-side critical section unless it
|
|
* can prove otherwise. Note that disabling of preemption (including
|
|
* disabling irqs) counts as an RCU-sched read-side critical section.
|
|
*/
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
{
|
|
int lockdep_opinion = 0;
|
|
|
|
if (debug_locks)
|
|
lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
|
|
return lockdep_opinion || preempt_count() != 0;
|
|
}
|
|
|
|
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
# define rcu_read_acquire() do { } while (0)
|
|
# define rcu_read_release() do { } while (0)
|
|
# define rcu_read_acquire_bh() do { } while (0)
|
|
# define rcu_read_release_bh() do { } while (0)
|
|
# define rcu_read_acquire_sched() do { } while (0)
|
|
# define rcu_read_release_sched() do { } while (0)
|
|
|
|
static inline int rcu_read_lock_held(void)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static inline int rcu_read_lock_bh_held(void)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
{
|
|
return preempt_count() != 0;
|
|
}
|
|
|
|
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
/**
|
|
* rcu_dereference_check - rcu_dereference with debug checking
|
|
*
|
|
* Do an rcu_dereference(), but check that the context is correct.
|
|
* For example, rcu_dereference_check(gp, rcu_read_lock_held()) to
|
|
* ensure that the rcu_dereference_check() executes within an RCU
|
|
* read-side critical section. It is also possible to check for
|
|
* locks being held, for example, by using lockdep_is_held().
|
|
*/
|
|
#define rcu_dereference_check(p, c) \
|
|
({ \
|
|
if (debug_locks) \
|
|
WARN_ON_ONCE(!(c)); \
|
|
rcu_dereference(p); \
|
|
})
|
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
#define rcu_dereference_check(p, c) rcu_dereference(p)
|
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
/**
|
|
* rcu_read_lock - mark the beginning of an RCU read-side critical section.
|
|
*
|
|
* When synchronize_rcu() is invoked on one CPU while other CPUs
|
|
* are within RCU read-side critical sections, then the
|
|
* synchronize_rcu() is guaranteed to block until after all the other
|
|
* CPUs exit their critical sections. Similarly, if call_rcu() is invoked
|
|
* on one CPU while other CPUs are within RCU read-side critical
|
|
* sections, invocation of the corresponding RCU callback is deferred
|
|
* until after the all the other CPUs exit their critical sections.
|
|
*
|
|
* Note, however, that RCU callbacks are permitted to run concurrently
|
|
* with RCU read-side critical sections. One way that this can happen
|
|
* is via the following sequence of events: (1) CPU 0 enters an RCU
|
|
* read-side critical section, (2) CPU 1 invokes call_rcu() to register
|
|
* an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
|
|
* (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
|
|
* callback is invoked. This is legal, because the RCU read-side critical
|
|
* section that was running concurrently with the call_rcu() (and which
|
|
* therefore might be referencing something that the corresponding RCU
|
|
* callback would free up) has completed before the corresponding
|
|
* RCU callback is invoked.
|
|
*
|
|
* RCU read-side critical sections may be nested. Any deferred actions
|
|
* will be deferred until the outermost RCU read-side critical section
|
|
* completes.
|
|
*
|
|
* It is illegal to block while in an RCU read-side critical section.
|
|
*/
|
|
static inline void rcu_read_lock(void)
|
|
{
|
|
__rcu_read_lock();
|
|
__acquire(RCU);
|
|
rcu_read_acquire();
|
|
}
|
|
|
|
/*
|
|
* So where is rcu_write_lock()? It does not exist, as there is no
|
|
* way for writers to lock out RCU readers. This is a feature, not
|
|
* a bug -- this property is what provides RCU's performance benefits.
|
|
* Of course, writers must coordinate with each other. The normal
|
|
* spinlock primitives work well for this, but any other technique may be
|
|
* used as well. RCU does not care how the writers keep out of each
|
|
* others' way, as long as they do so.
|
|
*/
|
|
|
|
/**
|
|
* rcu_read_unlock - marks the end of an RCU read-side critical section.
|
|
*
|
|
* See rcu_read_lock() for more information.
|
|
*/
|
|
static inline void rcu_read_unlock(void)
|
|
{
|
|
rcu_read_release();
|
|
__release(RCU);
|
|
__rcu_read_unlock();
|
|
}
|
|
|
|
/**
|
|
* rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section
|
|
*
|
|
* This is equivalent of rcu_read_lock(), but to be used when updates
|
|
* are being done using call_rcu_bh(). Since call_rcu_bh() callbacks
|
|
* consider completion of a softirq handler to be a quiescent state,
|
|
* a process in RCU read-side critical section must be protected by
|
|
* disabling softirqs. Read-side critical sections in interrupt context
|
|
* can use just rcu_read_lock().
|
|
*
|
|
*/
|
|
static inline void rcu_read_lock_bh(void)
|
|
{
|
|
__rcu_read_lock_bh();
|
|
__acquire(RCU_BH);
|
|
rcu_read_acquire_bh();
|
|
}
|
|
|
|
/*
|
|
* rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
|
|
*
|
|
* See rcu_read_lock_bh() for more information.
|
|
*/
|
|
static inline void rcu_read_unlock_bh(void)
|
|
{
|
|
rcu_read_release_bh();
|
|
__release(RCU_BH);
|
|
__rcu_read_unlock_bh();
|
|
}
|
|
|
|
/**
|
|
* rcu_read_lock_sched - mark the beginning of a RCU-classic critical section
|
|
*
|
|
* Should be used with either
|
|
* - synchronize_sched()
|
|
* or
|
|
* - call_rcu_sched() and rcu_barrier_sched()
|
|
* on the write-side to insure proper synchronization.
|
|
*/
|
|
static inline void rcu_read_lock_sched(void)
|
|
{
|
|
preempt_disable();
|
|
__acquire(RCU_SCHED);
|
|
rcu_read_acquire_sched();
|
|
}
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
|
static inline notrace void rcu_read_lock_sched_notrace(void)
|
|
{
|
|
preempt_disable_notrace();
|
|
__acquire(RCU_SCHED);
|
|
}
|
|
|
|
/*
|
|
* rcu_read_unlock_sched - marks the end of a RCU-classic critical section
|
|
*
|
|
* See rcu_read_lock_sched for more information.
|
|
*/
|
|
static inline void rcu_read_unlock_sched(void)
|
|
{
|
|
rcu_read_release_sched();
|
|
__release(RCU_SCHED);
|
|
preempt_enable();
|
|
}
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
|
static inline notrace void rcu_read_unlock_sched_notrace(void)
|
|
{
|
|
__release(RCU_SCHED);
|
|
preempt_enable_notrace();
|
|
}
|
|
|
|
|
|
/**
|
|
* rcu_dereference - fetch an RCU-protected pointer in an
|
|
* RCU read-side critical section. This pointer may later
|
|
* be safely dereferenced.
|
|
*
|
|
* Inserts memory barriers on architectures that require them
|
|
* (currently only the Alpha), and, more importantly, documents
|
|
* exactly which pointers are protected by RCU.
|
|
*/
|
|
|
|
#define rcu_dereference(p) ({ \
|
|
typeof(p) _________p1 = ACCESS_ONCE(p); \
|
|
smp_read_barrier_depends(); \
|
|
(_________p1); \
|
|
})
|
|
|
|
/**
|
|
* rcu_assign_pointer - assign (publicize) a pointer to a newly
|
|
* initialized structure that will be dereferenced by RCU read-side
|
|
* critical sections. Returns the value assigned.
|
|
*
|
|
* Inserts memory barriers on architectures that require them
|
|
* (pretty much all of them other than x86), and also prevents
|
|
* the compiler from reordering the code that initializes the
|
|
* structure after the pointer assignment. More importantly, this
|
|
* call documents which pointers will be dereferenced by RCU read-side
|
|
* code.
|
|
*/
|
|
|
|
#define rcu_assign_pointer(p, v) \
|
|
({ \
|
|
if (!__builtin_constant_p(v) || \
|
|
((v) != NULL)) \
|
|
smp_wmb(); \
|
|
(p) = (v); \
|
|
})
|
|
|
|
/* Infrastructure to implement the synchronize_() primitives. */
|
|
|
|
struct rcu_synchronize {
|
|
struct rcu_head head;
|
|
struct completion completion;
|
|
};
|
|
|
|
extern void wakeme_after_rcu(struct rcu_head *head);
|
|
|
|
/**
|
|
* call_rcu - Queue an RCU callback for invocation after a grace period.
|
|
* @head: structure to be used for queueing the RCU updates.
|
|
* @func: actual update function to be invoked after the grace period
|
|
*
|
|
* The update function will be invoked some time after a full grace
|
|
* period elapses, in other words after all currently executing RCU
|
|
* read-side critical sections have completed. RCU read-side critical
|
|
* sections are delimited by rcu_read_lock() and rcu_read_unlock(),
|
|
* and may be nested.
|
|
*/
|
|
extern void call_rcu(struct rcu_head *head,
|
|
void (*func)(struct rcu_head *head));
|
|
|
|
/**
|
|
* call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
|
|
* @head: structure to be used for queueing the RCU updates.
|
|
* @func: actual update function to be invoked after the grace period
|
|
*
|
|
* The update function will be invoked some time after a full grace
|
|
* period elapses, in other words after all currently executing RCU
|
|
* read-side critical sections have completed. call_rcu_bh() assumes
|
|
* that the read-side critical sections end on completion of a softirq
|
|
* handler. This means that read-side critical sections in process
|
|
* context must not be interrupted by softirqs. This interface is to be
|
|
* used when most of the read-side critical sections are in softirq context.
|
|
* RCU read-side critical sections are delimited by :
|
|
* - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
|
|
* OR
|
|
* - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
|
|
* These may be nested.
|
|
*/
|
|
extern void call_rcu_bh(struct rcu_head *head,
|
|
void (*func)(struct rcu_head *head));
|
|
|
|
#endif /* __LINUX_RCUPDATE_H */
|