2005-10-30 23:03:12 +00:00
/*
2006-06-27 09:54:02 +00:00
* Read - Copy Update module - based torture test facility
2005-10-30 23:03:12 +00:00
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation ; either version 2 of the License , or
* ( at your option ) any later version .
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
2013-12-03 18:02:52 +00:00
* along with this program ; if not , you can access it online at
* http : //www.gnu.org/licenses/gpl-2.0.html.
2005-10-30 23:03:12 +00:00
*
2006-10-04 09:17:13 +00:00
* Copyright ( C ) IBM Corporation , 2005 , 2006
2005-10-30 23:03:12 +00:00
*
* Authors : Paul E . McKenney < paulmck @ us . ibm . com >
2014-07-30 23:08:42 +00:00
* Josh Triplett < josh @ joshtriplett . org >
2005-10-30 23:03:12 +00:00
*
* See also : Documentation / RCU / torture . txt
*/
2018-05-15 19:25:05 +00:00
# define pr_fmt(fmt) fmt
2005-10-30 23:03:12 +00:00
# include <linux/types.h>
# include <linux/kernel.h>
# include <linux/init.h>
# include <linux/module.h>
# include <linux/kthread.h>
# include <linux/err.h>
# include <linux/spinlock.h>
# include <linux/smp.h>
# include <linux/rcupdate.h>
# include <linux/interrupt.h>
2017-02-02 18:15:33 +00:00
# include <linux/sched/signal.h>
2017-02-01 17:07:51 +00:00
# include <uapi/linux/sched/types.h>
2011-07-26 23:09:06 +00:00
# include <linux/atomic.h>
2005-10-30 23:03:12 +00:00
# include <linux/bitops.h>
# include <linux/completion.h>
# include <linux/moduleparam.h>
# include <linux/percpu.h>
# include <linux/notifier.h>
2008-12-16 00:13:07 +00:00
# include <linux/reboot.h>
2007-07-17 11:03:35 +00:00
# include <linux/freezer.h>
2005-10-30 23:03:12 +00:00
# include <linux/cpu.h>
# include <linux/delay.h>
# include <linux/stat.h>
2006-10-04 09:17:03 +00:00
# include <linux/srcu.h>
2008-04-29 07:59:25 +00:00
# include <linux/slab.h>
2012-11-15 00:26:40 +00:00
# include <linux/trace_clock.h>
2008-10-20 17:23:38 +00:00
# include <asm/byteorder.h>
2014-01-27 19:49:39 +00:00
# include <linux/torture.h>
2014-08-19 04:12:17 +00:00
# include <linux/vmalloc.h>
2017-08-30 17:40:17 +00:00
# include <linux/sched/debug.h>
2018-06-10 23:45:43 +00:00
# include <linux/sched/sysctl.h>
2005-10-30 23:03:12 +00:00
2017-05-03 16:51:55 +00:00
# include "rcu.h"
2005-10-30 23:03:12 +00:00
MODULE_LICENSE ( " GPL " ) ;
2014-07-30 23:08:42 +00:00
MODULE_AUTHOR ( " Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org> " ) ;
2005-10-30 23:03:12 +00:00
2013-10-09 03:23:47 +00:00
2018-05-25 14:29:25 +00:00
/* Bits for ->extendables field, extendables param, and related definitions. */
# define RCUTORTURE_RDR_SHIFT 8 /* Put SRCU index in upper bits. */
# define RCUTORTURE_RDR_MASK ((1 << RCUTORTURE_RDR_SHIFT) - 1)
2018-07-06 22:16:12 +00:00
# define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */
# define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */
# define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */
# define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */
# define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */
# define RCUTORTURE_RDR_RCU 0x20 /* ... entering another RCU reader. */
# define RCUTORTURE_RDR_NBITS 6 /* Number of bits defined above. */
# define RCUTORTURE_MAX_EXTEND \
( RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \
RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED )
2018-05-25 14:29:25 +00:00
# define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */
/* Must be power of two minus one. */
2014-08-19 04:12:17 +00:00
torture_param ( int , cbflood_inter_holdoff , HZ ,
" Holdoff between floods (jiffies) " ) ;
torture_param ( int , cbflood_intra_holdoff , 1 ,
" Holdoff between bursts (jiffies) " ) ;
torture_param ( int , cbflood_n_burst , 3 , " # bursts in flood, zero to disable " ) ;
torture_param ( int , cbflood_n_per_burst , 20000 ,
" # callbacks per burst in flood " ) ;
2018-05-25 14:29:25 +00:00
torture_param ( int , extendables , RCUTORTURE_MAX_EXTEND ,
" Extend readers by disabling bh (1), irqs (2), or preempt (4) " ) ;
2014-01-28 00:27:00 +00:00
torture_param ( int , fqs_duration , 0 ,
" Duration of fqs bursts (us), 0 to disable " ) ;
torture_param ( int , fqs_holdoff , 0 , " Holdoff time within fqs bursts (us) " ) ;
torture_param ( int , fqs_stutter , 3 , " Wait time between fqs bursts (s) " ) ;
2014-03-18 22:57:41 +00:00
torture_param ( bool , gp_cond , false , " Use conditional/async GP wait primitives " ) ;
2014-01-28 00:27:00 +00:00
torture_param ( bool , gp_exp , false , " Use expedited GP wait primitives " ) ;
torture_param ( bool , gp_normal , false ,
" Use normal (non-expedited) GP wait primitives " ) ;
2014-03-21 23:17:56 +00:00
torture_param ( bool , gp_sync , false , " Use synchronous GP wait primitives " ) ;
2014-01-28 00:27:00 +00:00
torture_param ( int , irqreader , 1 , " Allow RCU readers from irq handlers " ) ;
torture_param ( int , n_barrier_cbs , 0 ,
" # of callbacks/kthreads for barrier testing " ) ;
torture_param ( int , nfakewriters , 4 , " Number of RCU fake writer threads " ) ;
torture_param ( int , nreaders , - 1 , " Number of RCU reader threads " ) ;
torture_param ( int , object_debug , 0 ,
" Enable debug-object double call_rcu() testing " ) ;
torture_param ( int , onoff_holdoff , 0 , " Time after boot before CPU hotplugs (s) " ) ;
torture_param ( int , onoff_interval , 0 ,
2018-05-08 16:20:34 +00:00
" Time between CPU hotplugs (jiffies), 0=disable " ) ;
2014-01-28 00:27:00 +00:00
torture_param ( int , shuffle_interval , 3 , " Number of seconds between shuffles " ) ;
torture_param ( int , shutdown_secs , 0 , " Shutdown time (s), <= zero to disable. " ) ;
torture_param ( int , stall_cpu , 0 , " Stall duration (s), zero to disable. " ) ;
torture_param ( int , stall_cpu_holdoff , 10 ,
" Time to wait before starting stall (s). " ) ;
2017-08-18 23:11:37 +00:00
torture_param ( int , stall_cpu_irqsoff , 0 , " Disable interrupts while stalling. " ) ;
2014-01-28 00:27:00 +00:00
torture_param ( int , stat_interval , 60 ,
" Number of seconds between stats printk()s " ) ;
torture_param ( int , stutter , 5 , " Number of seconds to run/halt test " ) ;
torture_param ( int , test_boost , 1 , " Test RCU prio boost: 0=no, 1=maybe, 2=yes. " ) ;
torture_param ( int , test_boost_duration , 4 ,
" Duration of each boost test, seconds. " ) ;
torture_param ( int , test_boost_interval , 7 ,
" Interval between boost tests, seconds. " ) ;
torture_param ( bool , test_no_idle_hz , true ,
" Test support for tickless idle CPUs " ) ;
2018-05-09 17:29:18 +00:00
torture_param ( int , verbose , 1 ,
2014-01-30 21:38:09 +00:00
" Enable verbose debugging printk()s " ) ;
2014-01-28 00:27:00 +00:00
2013-06-13 22:12:24 +00:00
static char * torture_type = " rcu " ;
2007-03-06 09:42:13 +00:00
module_param ( torture_type , charp , 0444 ) ;
2013-06-13 22:12:24 +00:00
MODULE_PARM_DESC ( torture_type , " Type of RCU to torture (rcu, rcu_bh, ...) " ) ;
2005-10-30 23:03:12 +00:00
static int nrealreaders ;
2014-08-19 04:12:17 +00:00
static int ncbflooders ;
2005-10-30 23:03:12 +00:00
static struct task_struct * writer_task ;
2006-10-04 09:17:13 +00:00
static struct task_struct * * fakewriter_tasks ;
2005-10-30 23:03:12 +00:00
static struct task_struct * * reader_tasks ;
static struct task_struct * stats_task ;
2014-08-19 04:12:17 +00:00
static struct task_struct * * cbflood_task ;
2010-01-04 23:09:10 +00:00
static struct task_struct * fqs_task ;
2010-09-02 23:16:14 +00:00
static struct task_struct * boost_tasks [ NR_CPUS ] ;
2012-01-20 23:36:33 +00:00
static struct task_struct * stall_task ;
2012-02-21 01:51:45 +00:00
static struct task_struct * * barrier_cbs_tasks ;
static struct task_struct * barrier_task ;
2005-10-30 23:03:12 +00:00
# define RCU_TORTURE_PIPE_LEN 10
struct rcu_torture {
struct rcu_head rtort_rcu ;
int rtort_pipe_count ;
struct list_head rtort_free ;
2005-11-18 09:10:50 +00:00
int rtort_mbtest ;
2005-10-30 23:03:12 +00:00
} ;
static LIST_HEAD ( rcu_torture_freelist ) ;
2010-09-20 04:06:14 +00:00
static struct rcu_torture __rcu * rcu_torture_current ;
2011-04-04 04:33:51 +00:00
static unsigned long rcu_torture_current_version ;
2005-10-30 23:03:12 +00:00
static struct rcu_torture rcu_tortures [ 10 * RCU_TORTURE_PIPE_LEN ] ;
static DEFINE_SPINLOCK ( rcu_torture_lock ) ;
2016-03-01 16:52:19 +00:00
static DEFINE_PER_CPU ( long [ RCU_TORTURE_PIPE_LEN + 1 ] , rcu_torture_count ) ;
static DEFINE_PER_CPU ( long [ RCU_TORTURE_PIPE_LEN + 1 ] , rcu_torture_batch ) ;
2005-10-30 23:03:12 +00:00
static atomic_t rcu_torture_wcount [ RCU_TORTURE_PIPE_LEN + 1 ] ;
2006-10-04 09:17:03 +00:00
static atomic_t n_rcu_torture_alloc ;
static atomic_t n_rcu_torture_alloc_fail ;
static atomic_t n_rcu_torture_free ;
static atomic_t n_rcu_torture_mberror ;
static atomic_t n_rcu_torture_error ;
2012-02-21 01:51:45 +00:00
static long n_rcu_torture_barrier_error ;
2010-09-02 23:16:14 +00:00
static long n_rcu_torture_boost_ktrerror ;
static long n_rcu_torture_boost_rterror ;
static long n_rcu_torture_boost_failure ;
static long n_rcu_torture_boosts ;
2018-05-22 18:17:51 +00:00
static atomic_long_t n_rcu_torture_timers ;
2012-02-21 01:51:45 +00:00
static long n_barrier_attempts ;
2018-06-19 22:14:19 +00:00
static long n_barrier_successes ; /* did rcu_barrier test succeed? */
2014-08-19 04:12:17 +00:00
static atomic_long_t n_cbfloods ;
2006-10-04 09:17:14 +00:00
static struct list_head rcu_torture_removed ;
2008-06-18 12:21:44 +00:00
2014-02-19 18:51:42 +00:00
static int rcu_torture_writer_state ;
# define RTWS_FIXED_DELAY 0
# define RTWS_DELAY 1
# define RTWS_REPLACE 2
# define RTWS_DEF_FREE 3
# define RTWS_EXP_SYNC 4
2014-03-18 22:57:41 +00:00
# define RTWS_COND_GET 5
# define RTWS_COND_SYNC 6
2014-03-21 23:17:56 +00:00
# define RTWS_SYNC 7
# define RTWS_STUTTER 8
# define RTWS_STOPPING 9
2015-11-17 21:35:28 +00:00
static const char * const rcu_torture_writer_state_names [ ] = {
" RTWS_FIXED_DELAY " ,
" RTWS_DELAY " ,
" RTWS_REPLACE " ,
" RTWS_DEF_FREE " ,
" RTWS_EXP_SYNC " ,
" RTWS_COND_GET " ,
" RTWS_COND_SYNC " ,
" RTWS_SYNC " ,
" RTWS_STUTTER " ,
" RTWS_STOPPING " ,
} ;
static const char * rcu_torture_writer_state_getname ( void )
{
unsigned int i = READ_ONCE ( rcu_torture_writer_state ) ;
if ( i > = ARRAY_SIZE ( rcu_torture_writer_state_names ) )
return " ??? " ;
return rcu_torture_writer_state_names [ i ] ;
}
2014-02-19 18:51:42 +00:00
2011-04-18 06:45:23 +00:00
# if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
2010-09-02 23:16:14 +00:00
# define rcu_can_boost() 1
2011-04-18 06:45:23 +00:00
# else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
2010-09-02 23:16:14 +00:00
# define rcu_can_boost() 0
2011-04-18 06:45:23 +00:00
# endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
2010-09-02 23:16:14 +00:00
2013-02-04 18:36:13 +00:00
# ifdef CONFIG_RCU_TRACE
static u64 notrace rcu_trace_clock_local ( void )
{
u64 ts = trace_clock_local ( ) ;
2017-06-23 23:07:17 +00:00
( void ) do_div ( ts , NSEC_PER_USEC ) ;
2013-02-04 18:36:13 +00:00
return ts ;
}
# else /* #ifdef CONFIG_RCU_TRACE */
static u64 notrace rcu_trace_clock_local ( void )
{
return 0ULL ;
}
# endif /* #else #ifdef CONFIG_RCU_TRACE */
2010-09-02 23:16:14 +00:00
static unsigned long boost_starttime ; /* jiffies of next boost test start. */
2014-06-11 20:39:43 +00:00
static DEFINE_MUTEX ( boost_mutex ) ; /* protect setting boost_starttime */
2010-09-02 23:16:14 +00:00
/* and boost task create/destroy. */
2012-02-21 01:51:45 +00:00
static atomic_t barrier_cbs_count ; /* Barrier callbacks registered. */
2012-05-29 02:21:41 +00:00
static bool barrier_phase ; /* Test phase. */
2012-02-21 01:51:45 +00:00
static atomic_t barrier_cbs_invoked ; /* Barrier callbacks invoked. */
static wait_queue_head_t * barrier_cbs_wq ; /* Coordinate barrier testing. */
static DECLARE_WAIT_QUEUE_HEAD ( barrier_wq ) ;
2010-09-02 23:16:14 +00:00
2005-10-30 23:03:12 +00:00
/*
* Allocate an element from the rcu_tortures pool .
*/
2006-01-08 09:02:17 +00:00
static struct rcu_torture *
2005-10-30 23:03:12 +00:00
rcu_torture_alloc ( void )
{
struct list_head * p ;
2006-01-25 18:50:12 +00:00
spin_lock_bh ( & rcu_torture_lock ) ;
2005-10-30 23:03:12 +00:00
if ( list_empty ( & rcu_torture_freelist ) ) {
atomic_inc ( & n_rcu_torture_alloc_fail ) ;
2006-01-25 18:50:12 +00:00
spin_unlock_bh ( & rcu_torture_lock ) ;
2005-10-30 23:03:12 +00:00
return NULL ;
}
atomic_inc ( & n_rcu_torture_alloc ) ;
p = rcu_torture_freelist . next ;
list_del_init ( p ) ;
2006-01-25 18:50:12 +00:00
spin_unlock_bh ( & rcu_torture_lock ) ;
2005-10-30 23:03:12 +00:00
return container_of ( p , struct rcu_torture , rtort_free ) ;
}
/*
* Free an element to the rcu_tortures pool .
*/
static void
rcu_torture_free ( struct rcu_torture * p )
{
atomic_inc ( & n_rcu_torture_free ) ;
2006-01-25 18:50:12 +00:00
spin_lock_bh ( & rcu_torture_lock ) ;
2005-10-30 23:03:12 +00:00
list_add_tail ( & p - > rtort_free , & rcu_torture_freelist ) ;
2006-01-25 18:50:12 +00:00
spin_unlock_bh ( & rcu_torture_lock ) ;
2005-10-30 23:03:12 +00:00
}
2006-06-27 09:54:03 +00:00
/*
* Operations vector for selecting different types of tests .
*/
struct rcu_torture_ops {
2014-02-19 18:51:42 +00:00
int ttype ;
2006-06-27 09:54:03 +00:00
void ( * init ) ( void ) ;
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
void ( * cleanup ) ( void ) ;
2006-06-27 09:54:03 +00:00
int ( * readlock ) ( void ) ;
2014-01-27 19:49:39 +00:00
void ( * read_delay ) ( struct torture_random_state * rrsp ) ;
2006-06-27 09:54:03 +00:00
void ( * readunlock ) ( int idx ) ;
2018-04-27 18:39:34 +00:00
unsigned long ( * get_gp_seq ) ( void ) ;
2018-05-15 22:24:41 +00:00
unsigned long ( * gp_diff ) ( unsigned long new , unsigned long old ) ;
2009-06-25 16:08:17 +00:00
void ( * deferred_free ) ( struct rcu_torture * p ) ;
2006-10-04 09:17:13 +00:00
void ( * sync ) ( void ) ;
2013-06-12 22:12:21 +00:00
void ( * exp_sync ) ( void ) ;
2014-03-18 22:57:41 +00:00
unsigned long ( * get_state ) ( void ) ;
void ( * cond_sync ) ( unsigned long oldstate ) ;
2015-07-29 05:29:39 +00:00
call_rcu_func_t call ;
2008-05-12 19:21:05 +00:00
void ( * cb_barrier ) ( void ) ;
2010-01-04 23:09:10 +00:00
void ( * fqs ) ( void ) ;
2014-07-14 13:16:15 +00:00
void ( * stats ) ( void ) ;
2009-06-25 16:08:17 +00:00
int irq_capable ;
2010-09-02 23:16:14 +00:00
int can_boost ;
2018-05-25 14:29:25 +00:00
int extendables ;
int ext_irq_conflict ;
2013-07-12 20:50:28 +00:00
const char * name ;
2006-06-27 09:54:03 +00:00
} ;
2009-09-18 17:28:19 +00:00
static struct rcu_torture_ops * cur_ops ;
2006-06-27 09:54:03 +00:00
/*
* Definitions for rcu torture testing .
*/
2006-09-29 08:59:30 +00:00
static int rcu_torture_read_lock ( void ) __acquires ( RCU )
2006-06-27 09:54:03 +00:00
{
rcu_read_lock ( ) ;
return 0 ;
}
2014-01-27 19:49:39 +00:00
static void rcu_read_delay ( struct torture_random_state * rrsp )
2006-10-04 09:17:03 +00:00
{
2016-10-11 01:26:04 +00:00
unsigned long started ;
unsigned long completed ;
2009-09-08 22:54:35 +00:00
const unsigned long shortdelay_us = 200 ;
const unsigned long longdelay_ms = 50 ;
2016-10-11 01:26:04 +00:00
unsigned long long ts ;
2006-10-04 09:17:03 +00:00
2009-09-08 22:54:35 +00:00
/* We want a short delay sometimes to make a reader delay the grace
* period , and we want a long delay occasionally to trigger
* force_quiescent_state . */
2006-10-04 09:17:03 +00:00
2016-10-11 01:26:04 +00:00
if ( ! ( torture_random ( rrsp ) % ( nrealreaders * 2000 * longdelay_ms ) ) ) {
2018-04-27 18:39:34 +00:00
started = cur_ops - > get_gp_seq ( ) ;
2016-10-11 01:26:04 +00:00
ts = rcu_trace_clock_local ( ) ;
2009-09-08 22:54:35 +00:00
mdelay ( longdelay_ms ) ;
2018-04-27 18:39:34 +00:00
completed = cur_ops - > get_gp_seq ( ) ;
2016-10-11 01:26:04 +00:00
do_trace_rcu_torture_read ( cur_ops - > name , NULL , ts ,
started , completed ) ;
}
2014-01-27 19:49:39 +00:00
if ( ! ( torture_random ( rrsp ) % ( nrealreaders * 2 * shortdelay_us ) ) )
2009-09-08 22:54:35 +00:00
udelay ( shortdelay_us ) ;
2014-01-27 19:49:39 +00:00
if ( ! preempt_count ( ) & &
2017-10-16 18:23:42 +00:00
! ( torture_random ( rrsp ) % ( nrealreaders * 500 ) ) )
2017-10-16 18:05:03 +00:00
torture_preempt_schedule ( ) ; /* QS only if preemptible. */
2006-10-04 09:17:03 +00:00
}
2006-09-29 08:59:30 +00:00
static void rcu_torture_read_unlock ( int idx ) __releases ( RCU )
2006-06-27 09:54:03 +00:00
{
rcu_read_unlock ( ) ;
}
2014-03-18 22:57:41 +00:00
/*
* Update callback in the pipe . This should be invoked after a grace period .
*/
static bool
rcu_torture_pipe_update_one ( struct rcu_torture * rp )
{
int i ;
i = rp - > rtort_pipe_count ;
if ( i > RCU_TORTURE_PIPE_LEN )
i = RCU_TORTURE_PIPE_LEN ;
atomic_inc ( & rcu_torture_wcount [ i ] ) ;
if ( + + rp - > rtort_pipe_count > = RCU_TORTURE_PIPE_LEN ) {
rp - > rtort_mbtest = 0 ;
return true ;
}
return false ;
}
/*
* Update all callbacks in the pipe . Suitable for synchronous grace - period
* primitives .
*/
static void
rcu_torture_pipe_update ( struct rcu_torture * old_rp )
{
struct rcu_torture * rp ;
struct rcu_torture * rp1 ;
if ( old_rp )
list_add ( & old_rp - > rtort_free , & rcu_torture_removed ) ;
list_for_each_entry_safe ( rp , rp1 , & rcu_torture_removed , rtort_free ) {
if ( rcu_torture_pipe_update_one ( rp ) ) {
list_del ( & rp - > rtort_free ) ;
rcu_torture_free ( rp ) ;
}
}
}
2006-06-27 09:54:03 +00:00
static void
rcu_torture_cb ( struct rcu_head * p )
{
struct rcu_torture * rp = container_of ( p , struct rcu_torture , rtort_rcu ) ;
2014-01-30 23:49:29 +00:00
if ( torture_must_stop_irq ( ) ) {
2006-06-27 09:54:03 +00:00
/* Test is ending, just drop callbacks on the floor. */
/* The next initialization will pick up the pieces. */
return ;
}
2014-03-18 22:57:41 +00:00
if ( rcu_torture_pipe_update_one ( rp ) )
2006-06-27 09:54:03 +00:00
rcu_torture_free ( rp ) ;
2014-03-18 22:57:41 +00:00
else
2009-06-25 16:08:17 +00:00
cur_ops - > deferred_free ( rp ) ;
2006-06-27 09:54:03 +00:00
}
2014-11-21 22:19:26 +00:00
static unsigned long rcu_no_completed ( void )
2009-12-02 20:10:15 +00:00
{
return 0 ;
}
2006-06-27 09:54:03 +00:00
static void rcu_torture_deferred_free ( struct rcu_torture * p )
{
call_rcu ( & p - > rtort_rcu , rcu_torture_cb ) ;
}
2006-10-04 09:17:14 +00:00
static void rcu_sync_torture_init ( void )
{
INIT_LIST_HEAD ( & rcu_torture_removed ) ;
}
2013-06-12 22:12:21 +00:00
static struct rcu_torture_ops rcu_ops = {
2014-02-19 18:51:42 +00:00
. ttype = RCU_FLAVOR ,
2009-06-25 16:08:17 +00:00
. init = rcu_sync_torture_init ,
. readlock = rcu_torture_read_lock ,
. read_delay = rcu_read_delay ,
. readunlock = rcu_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = rcu_get_gp_seq ,
2018-05-15 22:24:41 +00:00
. gp_diff = rcu_seq_diff ,
2013-06-12 22:12:21 +00:00
. deferred_free = rcu_torture_deferred_free ,
2009-06-25 16:08:17 +00:00
. sync = synchronize_rcu ,
2013-06-12 22:12:21 +00:00
. exp_sync = synchronize_rcu_expedited ,
2014-03-18 22:57:41 +00:00
. get_state = get_state_synchronize_rcu ,
. cond_sync = cond_synchronize_rcu ,
2013-06-12 22:12:21 +00:00
. call = call_rcu ,
. cb_barrier = rcu_barrier ,
2010-01-04 23:09:10 +00:00
. fqs = rcu_force_quiescent_state ,
2009-12-02 20:10:15 +00:00
. stats = NULL ,
. irq_capable = 1 ,
2010-09-02 23:16:14 +00:00
. can_boost = rcu_can_boost ( ) ,
2018-06-21 23:17:46 +00:00
. extendables = RCUTORTURE_MAX_EXTEND ,
2013-06-12 22:12:21 +00:00
. name = " rcu "
2009-12-02 20:10:15 +00:00
} ;
2006-06-27 09:54:04 +00:00
/*
* Definitions for rcu_bh torture testing .
*/
2006-09-29 08:59:30 +00:00
static int rcu_bh_torture_read_lock ( void ) __acquires ( RCU_BH )
2006-06-27 09:54:04 +00:00
{
rcu_read_lock_bh ( ) ;
return 0 ;
}
2006-09-29 08:59:30 +00:00
static void rcu_bh_torture_read_unlock ( int idx ) __releases ( RCU_BH )
2006-06-27 09:54:04 +00:00
{
rcu_read_unlock_bh ( ) ;
}
static void rcu_bh_torture_deferred_free ( struct rcu_torture * p )
{
call_rcu_bh ( & p - > rtort_rcu , rcu_torture_cb ) ;
}
static struct rcu_torture_ops rcu_bh_ops = {
2014-02-19 18:51:42 +00:00
. ttype = RCU_BH_FLAVOR ,
2013-06-12 22:12:21 +00:00
. init = rcu_sync_torture_init ,
2009-06-25 16:08:17 +00:00
. readlock = rcu_bh_torture_read_lock ,
. read_delay = rcu_read_delay , /* just reuse rcu's version. */
. readunlock = rcu_bh_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = rcu_bh_get_gp_seq ,
2018-05-15 22:24:41 +00:00
. gp_diff = rcu_seq_diff ,
2009-06-25 16:08:17 +00:00
. deferred_free = rcu_bh_torture_deferred_free ,
2011-06-07 23:59:35 +00:00
. sync = synchronize_rcu_bh ,
2013-06-12 22:12:21 +00:00
. exp_sync = synchronize_rcu_bh_expedited ,
2012-02-21 01:51:45 +00:00
. call = call_rcu_bh ,
2009-06-25 16:08:17 +00:00
. cb_barrier = rcu_barrier_bh ,
2010-01-04 23:09:10 +00:00
. fqs = rcu_bh_force_quiescent_state ,
2009-06-25 16:08:17 +00:00
. stats = NULL ,
. irq_capable = 1 ,
2018-05-25 14:29:25 +00:00
. extendables = ( RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ ) ,
. ext_irq_conflict = RCUTORTURE_RDR_RCU ,
2009-06-25 16:08:17 +00:00
. name = " rcu_bh "
2006-06-27 09:54:04 +00:00
} ;
2014-02-06 16:45:56 +00:00
/*
* Don ' t even think about trying any of these in real life ! ! !
* The names includes " busted " , and they really means it !
* The only purpose of these functions is to provide a buggy RCU
* implementation to make sure that rcutorture correctly emits
* buggy - RCU error messages .
*/
static void rcu_busted_torture_deferred_free ( struct rcu_torture * p )
{
/* This is a deliberate bug for testing purposes only! */
rcu_torture_cb ( & p - > rtort_rcu ) ;
}
static void synchronize_rcu_busted ( void )
{
/* This is a deliberate bug for testing purposes only! */
}
static void
2015-07-29 05:29:38 +00:00
call_rcu_busted ( struct rcu_head * head , rcu_callback_t func )
2014-02-06 16:45:56 +00:00
{
/* This is a deliberate bug for testing purposes only! */
func ( head ) ;
}
static struct rcu_torture_ops rcu_busted_ops = {
2014-02-19 18:51:42 +00:00
. ttype = INVALID_RCU_FLAVOR ,
2014-02-06 16:45:56 +00:00
. init = rcu_sync_torture_init ,
. readlock = rcu_torture_read_lock ,
. read_delay = rcu_read_delay , /* just reuse rcu's version. */
. readunlock = rcu_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = rcu_no_completed ,
2014-02-06 16:45:56 +00:00
. deferred_free = rcu_busted_torture_deferred_free ,
. sync = synchronize_rcu_busted ,
. exp_sync = synchronize_rcu_busted ,
. call = call_rcu_busted ,
. cb_barrier = NULL ,
. fqs = NULL ,
. stats = NULL ,
. irq_capable = 1 ,
2017-06-06 23:39:00 +00:00
. name = " busted "
2014-02-06 16:45:56 +00:00
} ;
2006-10-04 09:17:03 +00:00
/*
* Definitions for srcu torture testing .
*/
2012-10-12 17:14:17 +00:00
DEFINE_STATIC_SRCU ( srcu_ctl ) ;
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
static struct srcu_struct srcu_ctld ;
static struct srcu_struct * srcu_ctlp = & srcu_ctl ;
2006-10-04 09:17:03 +00:00
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
static int srcu_torture_read_lock ( void ) __acquires ( srcu_ctlp )
2006-10-04 09:17:03 +00:00
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
return srcu_read_lock ( srcu_ctlp ) ;
2006-10-04 09:17:03 +00:00
}
2014-01-27 19:49:39 +00:00
static void srcu_read_delay ( struct torture_random_state * rrsp )
2006-10-04 09:17:03 +00:00
{
long delay ;
const long uspertick = 1000000 / HZ ;
const long longdelay = 10 ;
/* We want there to be long-running readers, but not all the time. */
2014-01-27 19:49:39 +00:00
delay = torture_random ( rrsp ) %
( nrealreaders * 2 * longdelay * uspertick ) ;
2017-06-06 19:52:44 +00:00
if ( ! delay & & in_task ( ) )
2006-10-04 09:17:03 +00:00
schedule_timeout_interruptible ( longdelay ) ;
2010-06-21 08:57:42 +00:00
else
rcu_read_delay ( rrsp ) ;
2006-10-04 09:17:03 +00:00
}
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
static void srcu_torture_read_unlock ( int idx ) __releases ( srcu_ctlp )
2006-10-04 09:17:03 +00:00
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
srcu_read_unlock ( srcu_ctlp , idx ) ;
2006-10-04 09:17:03 +00:00
}
2014-11-21 22:19:26 +00:00
static unsigned long srcu_torture_completed ( void )
2006-10-04 09:17:03 +00:00
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
return srcu_batches_completed ( srcu_ctlp ) ;
2006-10-04 09:17:03 +00:00
}
2012-03-19 08:12:14 +00:00
static void srcu_torture_deferred_free ( struct rcu_torture * rp )
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
call_srcu ( srcu_ctlp , & rp - > rtort_rcu , rcu_torture_cb ) ;
2012-03-19 08:12:14 +00:00
}
2006-10-04 09:17:13 +00:00
static void srcu_torture_synchronize ( void )
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
synchronize_srcu ( srcu_ctlp ) ;
2006-10-04 09:17:13 +00:00
}
2012-05-08 17:21:50 +00:00
static void srcu_torture_call ( struct rcu_head * head ,
2015-07-29 05:29:38 +00:00
rcu_callback_t func )
2012-05-08 17:21:50 +00:00
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
call_srcu ( srcu_ctlp , head , func ) ;
2012-05-08 17:21:50 +00:00
}
static void srcu_torture_barrier ( void )
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
srcu_barrier ( srcu_ctlp ) ;
2012-05-08 17:21:50 +00:00
}
2014-07-14 13:16:15 +00:00
static void srcu_torture_stats ( void )
2006-10-04 09:17:03 +00:00
{
2017-05-22 20:31:03 +00:00
srcu_torture_stats_print ( srcu_ctlp , torture_type , TORTURE_FLAG ) ;
2006-10-04 09:17:03 +00:00
}
2013-06-12 22:12:21 +00:00
static void srcu_torture_synchronize_expedited ( void )
{
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
synchronize_srcu_expedited ( srcu_ctlp ) ;
2013-06-12 22:12:21 +00:00
}
2006-10-04 09:17:03 +00:00
static struct rcu_torture_ops srcu_ops = {
2014-02-19 18:51:42 +00:00
. ttype = SRCU_FLAVOR ,
2012-10-12 17:14:17 +00:00
. init = rcu_sync_torture_init ,
2009-06-25 16:08:17 +00:00
. readlock = srcu_torture_read_lock ,
. read_delay = srcu_read_delay ,
. readunlock = srcu_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = srcu_torture_completed ,
2012-03-19 08:12:14 +00:00
. deferred_free = srcu_torture_deferred_free ,
2009-06-25 16:08:17 +00:00
. sync = srcu_torture_synchronize ,
2013-06-12 22:12:21 +00:00
. exp_sync = srcu_torture_synchronize_expedited ,
2012-05-08 17:21:50 +00:00
. call = srcu_torture_call ,
. cb_barrier = srcu_torture_barrier ,
2009-06-25 16:08:17 +00:00
. stats = srcu_torture_stats ,
2017-06-06 19:52:44 +00:00
. irq_capable = 1 ,
2009-06-25 16:08:17 +00:00
. name = " srcu "
2006-10-04 09:17:03 +00:00
} ;
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
static void srcu_torture_init ( void )
{
rcu_sync_torture_init ( ) ;
WARN_ON ( init_srcu_struct ( & srcu_ctld ) ) ;
srcu_ctlp = & srcu_ctld ;
}
static void srcu_torture_cleanup ( void )
{
2018-04-06 00:19:17 +00:00
static DEFINE_TORTURE_RANDOM ( rand ) ;
if ( torture_random ( & rand ) & 0x800 )
cleanup_srcu_struct ( & srcu_ctld ) ;
else
cleanup_srcu_struct_quiesced ( & srcu_ctld ) ;
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
srcu_ctlp = & srcu_ctl ; /* In case of a later rcutorture run. */
}
/* As above, but dynamically allocated. */
static struct rcu_torture_ops srcud_ops = {
. ttype = SRCU_FLAVOR ,
. init = srcu_torture_init ,
. cleanup = srcu_torture_cleanup ,
. readlock = srcu_torture_read_lock ,
. read_delay = srcu_read_delay ,
. readunlock = srcu_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = srcu_torture_completed ,
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
. deferred_free = srcu_torture_deferred_free ,
. sync = srcu_torture_synchronize ,
. exp_sync = srcu_torture_synchronize_expedited ,
. call = srcu_torture_call ,
. cb_barrier = srcu_torture_barrier ,
. stats = srcu_torture_stats ,
2017-06-06 19:52:44 +00:00
. irq_capable = 1 ,
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
. name = " srcud "
} ;
2018-05-25 14:29:25 +00:00
/* As above, but broken due to inappropriate reader extension. */
static struct rcu_torture_ops busted_srcud_ops = {
. ttype = SRCU_FLAVOR ,
. init = srcu_torture_init ,
. cleanup = srcu_torture_cleanup ,
. readlock = srcu_torture_read_lock ,
. read_delay = rcu_read_delay ,
. readunlock = srcu_torture_read_unlock ,
. get_gp_seq = srcu_torture_completed ,
. deferred_free = srcu_torture_deferred_free ,
. sync = srcu_torture_synchronize ,
. exp_sync = srcu_torture_synchronize_expedited ,
. call = srcu_torture_call ,
. cb_barrier = srcu_torture_barrier ,
. stats = srcu_torture_stats ,
. irq_capable = 1 ,
. extendables = RCUTORTURE_MAX_EXTEND ,
. name = " busted_srcud "
} ;
2006-10-04 09:17:16 +00:00
/*
* Definitions for sched torture testing .
*/
static int sched_torture_read_lock ( void )
{
preempt_disable ( ) ;
return 0 ;
}
static void sched_torture_read_unlock ( int idx )
{
preempt_enable ( ) ;
}
2008-05-12 19:21:05 +00:00
static void rcu_sched_torture_deferred_free ( struct rcu_torture * p )
{
call_rcu_sched ( & p - > rtort_rcu , rcu_torture_cb ) ;
}
2006-10-04 09:17:16 +00:00
static struct rcu_torture_ops sched_ops = {
2014-02-19 18:51:42 +00:00
. ttype = RCU_SCHED_FLAVOR ,
2009-06-25 16:08:17 +00:00
. init = rcu_sync_torture_init ,
. readlock = sched_torture_read_lock ,
. read_delay = rcu_read_delay , /* just reuse rcu's version. */
. readunlock = sched_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = rcu_sched_get_gp_seq ,
2018-05-15 22:24:41 +00:00
. gp_diff = rcu_seq_diff ,
2009-06-25 16:08:17 +00:00
. deferred_free = rcu_sched_torture_deferred_free ,
2011-06-07 23:59:35 +00:00
. sync = synchronize_sched ,
2013-06-12 22:12:21 +00:00
. exp_sync = synchronize_sched_expedited ,
2015-05-30 17:11:24 +00:00
. get_state = get_state_synchronize_sched ,
. cond_sync = cond_synchronize_sched ,
2013-06-12 22:12:21 +00:00
. call = call_rcu_sched ,
2009-06-25 16:08:17 +00:00
. cb_barrier = rcu_barrier_sched ,
2010-01-04 23:09:10 +00:00
. fqs = rcu_sched_force_quiescent_state ,
2009-06-25 16:08:17 +00:00
. stats = NULL ,
. irq_capable = 1 ,
2018-05-25 14:29:25 +00:00
. extendables = RCUTORTURE_MAX_EXTEND ,
2009-06-25 16:08:17 +00:00
. name = " sched "
2006-10-04 09:17:16 +00:00
} ;
2014-07-01 18:59:36 +00:00
/*
* Definitions for RCU - tasks torture testing .
*/
static int tasks_torture_read_lock ( void )
{
return 0 ;
}
static void tasks_torture_read_unlock ( int idx )
{
}
static void rcu_tasks_torture_deferred_free ( struct rcu_torture * p )
{
call_rcu_tasks ( & p - > rtort_rcu , rcu_torture_cb ) ;
}
static struct rcu_torture_ops tasks_ops = {
. ttype = RCU_TASKS_FLAVOR ,
. init = rcu_sync_torture_init ,
. readlock = tasks_torture_read_lock ,
. read_delay = rcu_read_delay , /* just reuse rcu's version. */
. readunlock = tasks_torture_read_unlock ,
2018-04-27 18:39:34 +00:00
. get_gp_seq = rcu_no_completed ,
2014-07-01 18:59:36 +00:00
. deferred_free = rcu_tasks_torture_deferred_free ,
. sync = synchronize_rcu_tasks ,
. exp_sync = synchronize_rcu_tasks ,
. call = call_rcu_tasks ,
. cb_barrier = rcu_barrier_tasks ,
. fqs = NULL ,
. stats = NULL ,
. irq_capable = 1 ,
. name = " tasks "
} ;
2018-05-15 22:24:41 +00:00
static unsigned long rcutorture_seq_diff ( unsigned long new , unsigned long old )
{
if ( ! cur_ops - > gp_diff )
return new - old ;
return cur_ops - > gp_diff ( new , old ) ;
}
2015-06-30 15:57:57 +00:00
static bool __maybe_unused torturing_tasks ( void )
{
return cur_ops = = & tasks_ops ;
}
2010-09-02 23:16:14 +00:00
/*
* RCU torture priority - boost testing . Runs one real - time thread per
* CPU for moderate bursts , repeatedly registering RCU callbacks and
* spinning waiting for them to be invoked . If a given callback takes
* too long to be invoked , we assume that priority inversion has occurred .
*/
struct rcu_boost_inflight {
struct rcu_head rcu ;
int inflight ;
} ;
static void rcu_torture_boost_cb ( struct rcu_head * head )
{
struct rcu_boost_inflight * rbip =
container_of ( head , struct rcu_boost_inflight , rcu ) ;
2015-04-13 18:58:08 +00:00
/* Ensure RCU-core accesses precede clearing ->inflight */
smp_store_release ( & rbip - > inflight , 0 ) ;
2010-09-02 23:16:14 +00:00
}
2018-06-10 23:45:43 +00:00
static int old_rt_runtime = - 1 ;
static void rcu_torture_disable_rt_throttle ( void )
{
/*
* Disable RT throttling so that rcutorture ' s boost threads don ' t get
* throttled . Only possible if rcutorture is built - in otherwise the
* user should manually do this by setting the sched_rt_period_us and
* sched_rt_runtime sysctls .
*/
if ( ! IS_BUILTIN ( CONFIG_RCU_TORTURE_TEST ) | | old_rt_runtime ! = - 1 )
return ;
old_rt_runtime = sysctl_sched_rt_runtime ;
sysctl_sched_rt_runtime = - 1 ;
}
static void rcu_torture_enable_rt_throttle ( void )
{
if ( ! IS_BUILTIN ( CONFIG_RCU_TORTURE_TEST ) | | old_rt_runtime = = - 1 )
return ;
sysctl_sched_rt_runtime = old_rt_runtime ;
old_rt_runtime = - 1 ;
}
2018-06-10 23:45:44 +00:00
static bool rcu_torture_boost_failed ( unsigned long start , unsigned long end )
{
if ( end - start > test_boost_duration * HZ - HZ / 2 ) {
VERBOSE_TOROUT_STRING ( " rcu_torture_boost boosting failed " ) ;
n_rcu_torture_boost_failure + + ;
return true ; /* failed */
}
return false ; /* passed */
}
2010-09-02 23:16:14 +00:00
static int rcu_torture_boost ( void * arg )
{
unsigned long call_rcu_time ;
unsigned long endtime ;
unsigned long oldstarttime ;
struct rcu_boost_inflight rbi = { . inflight = 0 } ;
struct sched_param sp ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_boost started " ) ;
2010-09-02 23:16:14 +00:00
/* Set real-time priority. */
sp . sched_priority = 1 ;
if ( sched_setscheduler ( current , SCHED_FIFO , & sp ) < 0 ) {
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_boost RT prio failed! " ) ;
2010-09-02 23:16:14 +00:00
n_rcu_torture_boost_rterror + + ;
}
2011-03-30 16:10:44 +00:00
init_rcu_head_on_stack ( & rbi . rcu ) ;
2010-09-02 23:16:14 +00:00
/* Each pass through the following loop does one boost-test cycle. */
do {
2018-06-10 23:45:44 +00:00
/* Track if the test failed already in this test interval? */
bool failed = false ;
/* Increment n_rcu_torture_boosts once per boost-test */
while ( ! kthread_should_stop ( ) ) {
if ( mutex_trylock ( & boost_mutex ) ) {
n_rcu_torture_boosts + + ;
mutex_unlock ( & boost_mutex ) ;
break ;
}
schedule_timeout_uninterruptible ( 1 ) ;
}
if ( kthread_should_stop ( ) )
goto checkwait ;
2010-09-02 23:16:14 +00:00
/* Wait for the next test interval. */
oldstarttime = boost_starttime ;
2011-08-17 19:39:34 +00:00
while ( ULONG_CMP_LT ( jiffies , oldstarttime ) ) {
2013-01-11 00:21:07 +00:00
schedule_timeout_interruptible ( oldstarttime - jiffies ) ;
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_boost " ) ;
2014-01-30 23:49:29 +00:00
if ( torture_must_stop ( ) )
2010-09-02 23:16:14 +00:00
goto checkwait ;
}
/* Do one boost-test interval. */
endtime = oldstarttime + test_boost_duration * HZ ;
call_rcu_time = jiffies ;
2011-08-17 19:39:34 +00:00
while ( ULONG_CMP_LT ( jiffies , endtime ) ) {
2010-09-02 23:16:14 +00:00
/* If we don't have a callback in flight, post one. */
2015-04-13 18:58:08 +00:00
if ( ! smp_load_acquire ( & rbi . inflight ) ) {
/* RCU core before ->inflight = 1. */
smp_store_release ( & rbi . inflight , 1 ) ;
2010-09-02 23:16:14 +00:00
call_rcu ( & rbi . rcu , rcu_torture_boost_cb ) ;
2018-06-10 23:45:44 +00:00
/* Check if the boost test failed */
failed = failed | |
rcu_torture_boost_failed ( call_rcu_time ,
jiffies ) ;
2010-09-02 23:16:14 +00:00
call_rcu_time = jiffies ;
}
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_boost " ) ;
2014-01-30 23:49:29 +00:00
if ( torture_must_stop ( ) )
2010-09-02 23:16:14 +00:00
goto checkwait ;
}
2018-06-10 23:45:44 +00:00
/*
* If boost never happened , then inflight will always be 1 , in
* this case the boost check would never happen in the above
* loop so do another one here .
*/
if ( ! failed & & smp_load_acquire ( & rbi . inflight ) )
rcu_torture_boost_failed ( call_rcu_time , jiffies ) ;
2010-09-02 23:16:14 +00:00
/*
* Set the start time of the next test interval .
* Yes , this is vulnerable to long delays , but such
* delays simply cause a false negative for the next
* interval . Besides , we are running at RT priority ,
* so delays should be relatively rare .
*/
2011-08-18 16:30:32 +00:00
while ( oldstarttime = = boost_starttime & &
! kthread_should_stop ( ) ) {
2010-09-02 23:16:14 +00:00
if ( mutex_trylock ( & boost_mutex ) ) {
boost_starttime = jiffies +
test_boost_interval * HZ ;
mutex_unlock ( & boost_mutex ) ;
break ;
}
schedule_timeout_uninterruptible ( 1 ) ;
}
/* Go do the stutter. */
2014-01-31 19:57:43 +00:00
checkwait : stutter_wait ( " rcu_torture_boost " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2010-09-02 23:16:14 +00:00
/* Clean up and exit. */
2015-04-13 18:58:08 +00:00
while ( ! kthread_should_stop ( ) | | smp_load_acquire ( & rbi . inflight ) ) {
2014-02-01 01:37:28 +00:00
torture_shutdown_absorb ( " rcu_torture_boost " ) ;
2010-09-02 23:16:14 +00:00
schedule_timeout_uninterruptible ( 1 ) ;
2014-02-01 01:37:28 +00:00
}
2011-06-21 08:48:03 +00:00
destroy_rcu_head_on_stack ( & rbi . rcu ) ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_boost " ) ;
2010-09-02 23:16:14 +00:00
return 0 ;
}
2014-08-19 04:12:17 +00:00
static void rcu_torture_cbflood_cb ( struct rcu_head * rhp )
{
}
/*
* RCU torture callback - flood kthread . Repeatedly induces bursts of calls
* to call_rcu ( ) or analogous , increasing the probability of occurrence
* of callback - overflow corner cases .
*/
static int
rcu_torture_cbflood ( void * arg )
{
int err = 1 ;
int i ;
int j ;
struct rcu_head * rhp ;
if ( cbflood_n_per_burst > 0 & &
cbflood_inter_holdoff > 0 & &
cbflood_intra_holdoff > 0 & &
cur_ops - > call & &
cur_ops - > cb_barrier ) {
treewide: Use array_size() in vmalloc()
The vmalloc() function has no 2-factor argument form, so multiplication
factors need to be wrapped in array_size(). This patch replaces cases of:
vmalloc(a * b)
with:
vmalloc(array_size(a, b))
as well as handling cases of:
vmalloc(a * b * c)
with:
vmalloc(array3_size(a, b, c))
This does, however, attempt to ignore constant size factors like:
vmalloc(4 * 1024)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
vmalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
vmalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
vmalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
vmalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
vmalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
vmalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
vmalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
vmalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
vmalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
vmalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
vmalloc(
- sizeof(TYPE) * (COUNT_ID)
+ array_size(COUNT_ID, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * COUNT_ID
+ array_size(COUNT_ID, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * (COUNT_CONST)
+ array_size(COUNT_CONST, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * COUNT_CONST
+ array_size(COUNT_CONST, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(THING) * (COUNT_ID)
+ array_size(COUNT_ID, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * COUNT_ID
+ array_size(COUNT_ID, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * (COUNT_CONST)
+ array_size(COUNT_CONST, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * COUNT_CONST
+ array_size(COUNT_CONST, sizeof(THING))
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
vmalloc(
- SIZE * COUNT
+ array_size(COUNT, SIZE)
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
vmalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
vmalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
vmalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
vmalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
vmalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
vmalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
vmalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
vmalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
vmalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
vmalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
vmalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
vmalloc(C1 * C2 * C3, ...)
|
vmalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants.
@@
expression E1, E2;
constant C1, C2;
@@
(
vmalloc(C1 * C2, ...)
|
vmalloc(
- E1 * E2
+ array_size(E1, E2)
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:27:11 +00:00
rhp = vmalloc ( array3_size ( cbflood_n_burst ,
cbflood_n_per_burst ,
sizeof ( * rhp ) ) ) ;
2014-08-19 04:12:17 +00:00
err = ! rhp ;
}
if ( err ) {
VERBOSE_TOROUT_STRING ( " rcu_torture_cbflood disabled: Bad args or OOM " ) ;
2015-06-23 01:11:31 +00:00
goto wait_for_stop ;
2014-08-19 04:12:17 +00:00
}
VERBOSE_TOROUT_STRING ( " rcu_torture_cbflood task started " ) ;
do {
schedule_timeout_interruptible ( cbflood_inter_holdoff ) ;
atomic_long_inc ( & n_cbfloods ) ;
WARN_ON ( signal_pending ( current ) ) ;
for ( i = 0 ; i < cbflood_n_burst ; i + + ) {
for ( j = 0 ; j < cbflood_n_per_burst ; j + + ) {
cur_ops - > call ( & rhp [ i * cbflood_n_per_burst + j ] ,
rcu_torture_cbflood_cb ) ;
}
schedule_timeout_interruptible ( cbflood_intra_holdoff ) ;
WARN_ON ( signal_pending ( current ) ) ;
}
cur_ops - > cb_barrier ( ) ;
stutter_wait ( " rcu_torture_cbflood " ) ;
} while ( ! torture_must_stop ( ) ) ;
2014-10-27 22:52:04 +00:00
vfree ( rhp ) ;
2015-06-23 01:11:31 +00:00
wait_for_stop :
2014-08-19 04:12:17 +00:00
torture_kthread_stopping ( " rcu_torture_cbflood " ) ;
return 0 ;
}
2010-01-04 23:09:10 +00:00
/*
* RCU torture force - quiescent - state kthread . Repeatedly induces
* bursts of calls to force_quiescent_state ( ) , increasing the probability
* of occurrence of some important types of race conditions .
*/
static int
rcu_torture_fqs ( void * arg )
{
unsigned long fqs_resume_time ;
int fqs_burst_remaining ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_fqs task started " ) ;
2010-01-04 23:09:10 +00:00
do {
fqs_resume_time = jiffies + fqs_stutter * HZ ;
2011-08-17 19:39:34 +00:00
while ( ULONG_CMP_LT ( jiffies , fqs_resume_time ) & &
! kthread_should_stop ( ) ) {
2010-01-04 23:09:10 +00:00
schedule_timeout_interruptible ( 1 ) ;
}
fqs_burst_remaining = fqs_duration ;
2011-08-17 19:39:34 +00:00
while ( fqs_burst_remaining > 0 & &
! kthread_should_stop ( ) ) {
2010-01-04 23:09:10 +00:00
cur_ops - > fqs ( ) ;
udelay ( fqs_holdoff ) ;
fqs_burst_remaining - = fqs_holdoff ;
}
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_fqs " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_fqs " ) ;
2010-01-04 23:09:10 +00:00
return 0 ;
}
2005-10-30 23:03:12 +00:00
/*
* RCU torture writer kthread . Repeatedly substitutes a new structure
* for that pointed to by rcu_torture_current , freeing the old structure
* after a series of grace periods ( the " pipeline " ) .
*/
static int
rcu_torture_writer ( void * arg )
{
2016-01-01 02:11:47 +00:00
bool can_expedite = ! rcu_gp_is_expedited ( ) & & ! rcu_gp_is_normal ( ) ;
2015-02-19 00:31:29 +00:00
int expediting = 0 ;
2014-03-18 22:57:41 +00:00
unsigned long gp_snap ;
bool gp_cond1 = gp_cond , gp_exp1 = gp_exp , gp_normal1 = gp_normal ;
2014-03-21 23:17:56 +00:00
bool gp_sync1 = gp_sync ;
2005-10-30 23:03:12 +00:00
int i ;
struct rcu_torture * rp ;
struct rcu_torture * old_rp ;
2014-01-27 19:49:39 +00:00
static DEFINE_TORTURE_RANDOM ( rand ) ;
2014-03-21 23:17:56 +00:00
int synctype [ ] = { RTWS_DEF_FREE , RTWS_EXP_SYNC ,
RTWS_COND_GET , RTWS_SYNC } ;
2014-03-18 22:57:41 +00:00
int nsynctypes = 0 ;
2005-10-30 23:03:12 +00:00
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_writer task started " ) ;
2017-12-08 19:37:24 +00:00
if ( ! can_expedite )
2016-01-01 00:27:06 +00:00
pr_alert ( " %s " TORTURE_FLAG
2017-12-08 19:37:24 +00:00
" GP expediting controlled from boot/sysfs for %s. \n " ,
2016-01-01 00:27:06 +00:00
torture_type , cur_ops - > name ) ;
2005-11-14 00:07:22 +00:00
2014-03-18 22:57:41 +00:00
/* Initialize synctype[] array. If none set, take default. */
2015-02-19 20:15:19 +00:00
if ( ! gp_cond1 & & ! gp_exp1 & & ! gp_normal1 & & ! gp_sync1 )
2014-03-21 23:17:56 +00:00
gp_cond1 = gp_exp1 = gp_normal1 = gp_sync1 = true ;
2017-12-08 20:23:10 +00:00
if ( gp_cond1 & & cur_ops - > get_state & & cur_ops - > cond_sync ) {
2014-03-18 22:57:41 +00:00
synctype [ nsynctypes + + ] = RTWS_COND_GET ;
2017-12-08 20:23:10 +00:00
pr_info ( " %s: Testing conditional GPs. \n " , __func__ ) ;
} else if ( gp_cond & & ( ! cur_ops - > get_state | | ! cur_ops - > cond_sync ) ) {
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: gp_cond without primitives. \n " , __func__ ) ;
2017-12-08 20:23:10 +00:00
}
if ( gp_exp1 & & cur_ops - > exp_sync ) {
2014-03-18 22:57:41 +00:00
synctype [ nsynctypes + + ] = RTWS_EXP_SYNC ;
2017-12-08 20:23:10 +00:00
pr_info ( " %s: Testing expedited GPs. \n " , __func__ ) ;
} else if ( gp_exp & & ! cur_ops - > exp_sync ) {
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: gp_exp without primitives. \n " , __func__ ) ;
2017-12-08 20:23:10 +00:00
}
if ( gp_normal1 & & cur_ops - > deferred_free ) {
2014-03-18 22:57:41 +00:00
synctype [ nsynctypes + + ] = RTWS_DEF_FREE ;
2017-12-08 20:23:10 +00:00
pr_info ( " %s: Testing asynchronous GPs. \n " , __func__ ) ;
} else if ( gp_normal & & ! cur_ops - > deferred_free ) {
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: gp_normal without primitives. \n " , __func__ ) ;
2017-12-08 20:23:10 +00:00
}
if ( gp_sync1 & & cur_ops - > sync ) {
2014-03-21 23:17:56 +00:00
synctype [ nsynctypes + + ] = RTWS_SYNC ;
2017-12-08 20:23:10 +00:00
pr_info ( " %s: Testing normal GPs. \n " , __func__ ) ;
} else if ( gp_sync & & ! cur_ops - > sync ) {
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: gp_sync without primitives. \n " , __func__ ) ;
2017-12-08 20:23:10 +00:00
}
2014-03-18 22:57:41 +00:00
if ( WARN_ONCE ( nsynctypes = = 0 ,
" rcu_torture_writer: No update-side primitives. \n " ) ) {
2014-03-21 23:17:56 +00:00
/*
* No updates primitives , so don ' t try updating .
* The resulting test won ' t be testing much , hence the
* above WARN_ONCE ( ) .
*/
2014-03-18 22:57:41 +00:00
rcu_torture_writer_state = RTWS_STOPPING ;
torture_kthread_stopping ( " rcu_torture_writer " ) ;
}
2005-10-30 23:03:12 +00:00
do {
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_FIXED_DELAY ;
2005-10-30 23:03:12 +00:00
schedule_timeout_uninterruptible ( 1 ) ;
2009-09-18 17:28:19 +00:00
rp = rcu_torture_alloc ( ) ;
if ( rp = = NULL )
2005-10-30 23:03:12 +00:00
continue ;
rp - > rtort_pipe_count = 0 ;
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_DELAY ;
2014-01-27 19:49:39 +00:00
udelay ( torture_random ( & rand ) & 0x3ff ) ;
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_REPLACE ;
2010-09-20 04:06:14 +00:00
old_rp = rcu_dereference_check ( rcu_torture_current ,
current = = writer_task ) ;
2005-11-18 09:10:50 +00:00
rp - > rtort_mbtest = 1 ;
2005-10-30 23:03:12 +00:00
rcu_assign_pointer ( rcu_torture_current , rp ) ;
2009-09-23 16:50:43 +00:00
smp_wmb ( ) ; /* Mods to old_rp must follow rcu_assign_pointer() */
2007-05-08 07:33:20 +00:00
if ( old_rp ) {
2005-10-30 23:03:12 +00:00
i = old_rp - > rtort_pipe_count ;
if ( i > RCU_TORTURE_PIPE_LEN )
i = RCU_TORTURE_PIPE_LEN ;
atomic_inc ( & rcu_torture_wcount [ i ] ) ;
old_rp - > rtort_pipe_count + + ;
2014-03-18 22:57:41 +00:00
switch ( synctype [ torture_random ( & rand ) % nsynctypes ] ) {
case RTWS_DEF_FREE :
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_DEF_FREE ;
2013-06-12 22:12:21 +00:00
cur_ops - > deferred_free ( old_rp ) ;
2014-03-18 22:57:41 +00:00
break ;
case RTWS_EXP_SYNC :
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_EXP_SYNC ;
2013-06-12 22:12:21 +00:00
cur_ops - > exp_sync ( ) ;
2014-03-18 22:57:41 +00:00
rcu_torture_pipe_update ( old_rp ) ;
break ;
case RTWS_COND_GET :
rcu_torture_writer_state = RTWS_COND_GET ;
gp_snap = cur_ops - > get_state ( ) ;
i = torture_random ( & rand ) % 16 ;
if ( i ! = 0 )
schedule_timeout_interruptible ( i ) ;
udelay ( torture_random ( & rand ) % 1000 ) ;
rcu_torture_writer_state = RTWS_COND_SYNC ;
cur_ops - > cond_sync ( gp_snap ) ;
rcu_torture_pipe_update ( old_rp ) ;
break ;
2014-03-21 23:17:56 +00:00
case RTWS_SYNC :
rcu_torture_writer_state = RTWS_SYNC ;
cur_ops - > sync ( ) ;
rcu_torture_pipe_update ( old_rp ) ;
break ;
2014-03-18 22:57:41 +00:00
default :
WARN_ON_ONCE ( 1 ) ;
break ;
2013-06-12 22:12:21 +00:00
}
2005-10-30 23:03:12 +00:00
}
2018-05-16 22:30:36 +00:00
rcu_torture_current_version + + ;
2015-02-19 00:31:29 +00:00
/* Cycle through nesting levels of rcu_expedite_gp() calls. */
if ( can_expedite & &
! ( torture_random ( & rand ) & 0xff & ( ! ! expediting - 1 ) ) ) {
WARN_ON_ONCE ( expediting = = 0 & & rcu_gp_is_expedited ( ) ) ;
if ( expediting > = 0 )
rcu_expedite_gp ( ) ;
else
rcu_unexpedite_gp ( ) ;
if ( + + expediting > 3 )
expediting = - expediting ;
2017-12-08 19:37:24 +00:00
} else if ( ! can_expedite ) { /* Disabled during boot, recheck. */
can_expedite = ! rcu_gp_is_expedited ( ) & &
! rcu_gp_is_normal ( ) ;
2015-02-19 00:31:29 +00:00
}
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_STUTTER ;
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_writer " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2015-02-19 00:31:29 +00:00
/* Reset expediting back to unexpedited. */
if ( expediting > 0 )
expediting = - expediting ;
while ( can_expedite & & expediting + + < 0 )
rcu_unexpedite_gp ( ) ;
WARN_ON_ONCE ( can_expedite & & rcu_gp_is_expedited ( ) ) ;
2017-12-08 19:37:24 +00:00
if ( ! can_expedite )
pr_alert ( " %s " TORTURE_FLAG
" Dynamic grace-period expediting was disabled. \n " ,
torture_type ) ;
2014-02-19 18:51:42 +00:00
rcu_torture_writer_state = RTWS_STOPPING ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_writer " ) ;
2005-10-30 23:03:12 +00:00
return 0 ;
}
2006-10-04 09:17:13 +00:00
/*
* RCU torture fake writer kthread . Repeatedly calls sync , with a random
* delay between calls .
*/
static int
rcu_torture_fakewriter ( void * arg )
{
2014-01-27 19:49:39 +00:00
DEFINE_TORTURE_RANDOM ( rand ) ;
2006-10-04 09:17:13 +00:00
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_fakewriter task started " ) ;
2014-02-11 07:34:49 +00:00
set_user_nice ( current , MAX_NICE ) ;
2006-10-04 09:17:13 +00:00
do {
2014-01-27 19:49:39 +00:00
schedule_timeout_uninterruptible ( 1 + torture_random ( & rand ) % 10 ) ;
udelay ( torture_random ( & rand ) & 0x3ff ) ;
2012-05-30 00:50:51 +00:00
if ( cur_ops - > cb_barrier ! = NULL & &
2014-01-27 19:49:39 +00:00
torture_random ( & rand ) % ( nfakewriters * 8 ) = = 0 ) {
2012-05-30 00:50:51 +00:00
cur_ops - > cb_barrier ( ) ;
2013-06-12 22:12:21 +00:00
} else if ( gp_normal = = gp_exp ) {
2017-12-08 18:48:41 +00:00
if ( cur_ops - > sync & & torture_random ( & rand ) & 0x80 )
2013-06-12 22:12:21 +00:00
cur_ops - > sync ( ) ;
2017-12-08 18:48:41 +00:00
else if ( cur_ops - > exp_sync )
2013-06-12 22:12:21 +00:00
cur_ops - > exp_sync ( ) ;
2017-12-08 18:48:41 +00:00
} else if ( gp_normal & & cur_ops - > sync ) {
2012-05-30 00:50:51 +00:00
cur_ops - > sync ( ) ;
2017-12-08 18:48:41 +00:00
} else if ( cur_ops - > exp_sync ) {
2013-06-12 22:12:21 +00:00
cur_ops - > exp_sync ( ) ;
}
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_fakewriter " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2006-10-04 09:17:13 +00:00
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_fakewriter " ) ;
2006-10-04 09:17:13 +00:00
return 0 ;
}
2017-07-20 22:27:32 +00:00
static void rcu_torture_timer_cb ( struct rcu_head * rhp )
{
kfree ( rhp ) ;
}
2008-06-25 19:24:52 +00:00
/*
2018-05-25 14:29:25 +00:00
* Do one extension of an RCU read - side critical section using the
* current reader state in readstate ( set to zero for initial entry
* to extended critical section ) , set the new state as specified by
* newstate ( set to zero for final exit from extended critical section ) ,
* and random - number - generator state in trsp . If this is neither the
* beginning or end of the critical section and if there was actually a
* change , do a - > read_delay ( ) .
2008-06-25 19:24:52 +00:00
*/
2018-05-25 14:29:25 +00:00
static void rcutorture_one_extend ( int * readstate , int newstate ,
struct torture_random_state * trsp )
{
int idxnew = - 1 ;
int idxold = * readstate ;
int statesnew = ~ * readstate & newstate ;
int statesold = * readstate & ~ newstate ;
WARN_ON_ONCE ( idxold < 0 ) ;
WARN_ON_ONCE ( ( idxold > > RCUTORTURE_RDR_SHIFT ) > 1 ) ;
/* First, put new protection in place to avoid critical-section gap. */
if ( statesnew & RCUTORTURE_RDR_BH )
local_bh_disable ( ) ;
if ( statesnew & RCUTORTURE_RDR_IRQ )
local_irq_disable ( ) ;
if ( statesnew & RCUTORTURE_RDR_PREEMPT )
preempt_disable ( ) ;
2018-07-06 22:16:12 +00:00
if ( statesnew & RCUTORTURE_RDR_RBH )
rcu_read_lock_bh ( ) ;
if ( statesnew & RCUTORTURE_RDR_SCHED )
rcu_read_lock_sched ( ) ;
2018-05-25 14:29:25 +00:00
if ( statesnew & RCUTORTURE_RDR_RCU )
idxnew = cur_ops - > readlock ( ) < < RCUTORTURE_RDR_SHIFT ;
/* Next, remove old protection, irq first due to bh conflict. */
if ( statesold & RCUTORTURE_RDR_IRQ )
local_irq_enable ( ) ;
if ( statesold & RCUTORTURE_RDR_BH )
local_bh_enable ( ) ;
if ( statesold & RCUTORTURE_RDR_PREEMPT )
preempt_enable ( ) ;
2018-07-06 22:16:12 +00:00
if ( statesold & RCUTORTURE_RDR_RBH )
rcu_read_unlock_bh ( ) ;
if ( statesold & RCUTORTURE_RDR_SCHED )
rcu_read_unlock_sched ( ) ;
2018-05-25 14:29:25 +00:00
if ( statesold & RCUTORTURE_RDR_RCU )
cur_ops - > readunlock ( idxold > > RCUTORTURE_RDR_SHIFT ) ;
/* Delay if neither beginning nor end and there was a change. */
if ( ( statesnew | | statesold ) & & * readstate & & newstate )
cur_ops - > read_delay ( trsp ) ;
/* Update the reader state. */
if ( idxnew = = - 1 )
idxnew = idxold & ~ RCUTORTURE_RDR_MASK ;
WARN_ON_ONCE ( idxnew < 0 ) ;
WARN_ON_ONCE ( ( idxnew > > RCUTORTURE_RDR_SHIFT ) > 1 ) ;
* readstate = idxnew | newstate ;
WARN_ON_ONCE ( ( * readstate > > RCUTORTURE_RDR_SHIFT ) < 0 ) ;
WARN_ON_ONCE ( ( * readstate > > RCUTORTURE_RDR_SHIFT ) > 1 ) ;
}
/* Return the biggest extendables mask given current RCU and boot parameters. */
static int rcutorture_extend_mask_max ( void )
{
int mask ;
WARN_ON_ONCE ( extendables & ~ RCUTORTURE_MAX_EXTEND ) ;
mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops - > extendables ;
mask = mask | RCUTORTURE_RDR_RCU ;
return mask ;
}
/* Return a random protection state mask, but with at least one bit set. */
static int
rcutorture_extend_mask ( int oldmask , struct torture_random_state * trsp )
{
int mask = rcutorture_extend_mask_max ( ) ;
2018-06-10 15:50:09 +00:00
unsigned long randmask1 = torture_random ( trsp ) > > 8 ;
unsigned long randmask2 = randmask1 > > 1 ;
2018-05-25 14:29:25 +00:00
WARN_ON_ONCE ( mask > > RCUTORTURE_RDR_SHIFT ) ;
2018-06-10 15:50:09 +00:00
/* Half the time lots of bits, half the time only one bit. */
if ( randmask1 & 0x1 )
mask = mask & randmask2 ;
else
mask = mask & ( 1 < < ( randmask2 % RCUTORTURE_RDR_NBITS ) ) ;
2018-07-06 22:16:12 +00:00
/* Can't enable bh w/irq disabled. */
2018-05-25 14:29:25 +00:00
if ( ( mask & RCUTORTURE_RDR_IRQ ) & &
2018-07-06 22:16:12 +00:00
( ( ! ( mask & RCUTORTURE_RDR_BH ) & & ( oldmask & RCUTORTURE_RDR_BH ) ) | |
( ! ( mask & RCUTORTURE_RDR_RBH ) & & ( oldmask & RCUTORTURE_RDR_RBH ) ) ) )
mask | = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH ;
2018-05-25 14:29:25 +00:00
if ( ( mask & RCUTORTURE_RDR_IRQ ) & &
! ( mask & cur_ops - > ext_irq_conflict ) & &
( oldmask & cur_ops - > ext_irq_conflict ) )
mask | = cur_ops - > ext_irq_conflict ; /* Or if readers object. */
return mask ? : RCUTORTURE_RDR_RCU ;
}
/*
* Do a randomly selected number of extensions of an existing RCU read - side
* critical section .
*/
static void rcutorture_loop_extend ( int * readstate ,
struct torture_random_state * trsp )
{
int i ;
int mask = rcutorture_extend_mask_max ( ) ;
WARN_ON_ONCE ( ! * readstate ) ; /* -Existing- RCU read-side critsect! */
if ( ! ( ( mask - 1 ) & mask ) )
return ; /* Current RCU flavor not extendable. */
i = ( torture_random ( trsp ) > > 3 ) & RCUTORTURE_RDR_MAX_LOOPS ;
while ( i - - ) {
mask = rcutorture_extend_mask ( * readstate , trsp ) ;
rcutorture_one_extend ( readstate , mask , trsp ) ;
}
}
2018-05-22 17:56:05 +00:00
/*
* Do one read - side critical section , returning false if there was
* no data to read . Can be invoked both from process context and
* from a timer handler .
*/
static bool rcu_torture_one_read ( struct torture_random_state * trsp )
2008-06-25 19:24:52 +00:00
{
2014-11-22 01:10:16 +00:00
unsigned long started ;
2014-11-21 22:19:26 +00:00
unsigned long completed ;
2018-05-25 14:29:25 +00:00
int newstate ;
2008-06-25 19:24:52 +00:00
struct rcu_torture * p ;
int pipe_count ;
2018-05-25 14:29:25 +00:00
int readstate = 0 ;
2012-11-15 00:26:40 +00:00
unsigned long long ts ;
2008-06-25 19:24:52 +00:00
2018-05-25 14:29:25 +00:00
newstate = rcutorture_extend_mask ( readstate , trsp ) ;
rcutorture_one_extend ( & readstate , newstate , trsp ) ;
2018-04-27 18:39:34 +00:00
started = cur_ops - > get_gp_seq ( ) ;
2013-02-04 18:36:13 +00:00
ts = rcu_trace_clock_local ( ) ;
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
p = rcu_dereference_check ( rcu_torture_current ,
rcu_read_lock_bh_held ( ) | |
rcu_read_lock_sched_held ( ) | |
2015-06-30 15:57:57 +00:00
srcu_read_lock_held ( srcu_ctlp ) | |
torturing_tasks ( ) ) ;
2008-06-25 19:24:52 +00:00
if ( p = = NULL ) {
2018-05-22 17:56:05 +00:00
/* Wait for rcu_torture_writer to get underway */
2018-05-25 14:29:25 +00:00
rcutorture_one_extend ( & readstate , 0 , trsp ) ;
2018-05-22 17:56:05 +00:00
return false ;
2008-06-25 19:24:52 +00:00
}
if ( p - > rtort_mbtest = = 0 )
atomic_inc ( & n_rcu_torture_mberror ) ;
2018-05-25 14:29:25 +00:00
rcutorture_loop_extend ( & readstate , trsp ) ;
2008-06-25 19:24:52 +00:00
preempt_disable ( ) ;
pipe_count = p - > rtort_pipe_count ;
if ( pipe_count > RCU_TORTURE_PIPE_LEN ) {
/* Should not happen, but... */
pipe_count = RCU_TORTURE_PIPE_LEN ;
}
2018-04-27 18:39:34 +00:00
completed = cur_ops - > get_gp_seq ( ) ;
2012-11-15 00:26:40 +00:00
if ( pipe_count > 1 ) {
2018-05-22 17:56:05 +00:00
do_trace_rcu_torture_read ( cur_ops - > name , & p - > rtort_rcu ,
ts , started , completed ) ;
2016-03-22 02:46:04 +00:00
rcu_ftrace_dump ( DUMP_ALL ) ;
2012-11-15 00:26:40 +00:00
}
2009-10-29 13:34:15 +00:00
__this_cpu_inc ( rcu_torture_count [ pipe_count ] ) ;
2018-05-15 22:24:41 +00:00
completed = rcutorture_seq_diff ( completed , started ) ;
2008-06-25 19:24:52 +00:00
if ( completed > RCU_TORTURE_PIPE_LEN ) {
/* Should not happen, but... */
completed = RCU_TORTURE_PIPE_LEN ;
}
2009-10-29 13:34:15 +00:00
__this_cpu_inc ( rcu_torture_batch [ completed ] ) ;
2008-06-25 19:24:52 +00:00
preempt_enable ( ) ;
2018-05-25 14:29:25 +00:00
rcutorture_one_extend ( & readstate , 0 , trsp ) ;
WARN_ON_ONCE ( readstate & RCUTORTURE_RDR_MASK ) ;
2018-05-22 17:56:05 +00:00
return true ;
}
2018-05-22 18:38:47 +00:00
static DEFINE_TORTURE_RANDOM_PERCPU ( rcu_torture_timer_rand ) ;
2008-06-25 19:24:52 +00:00
/*
* RCU torture reader from timer handler . Dereferences rcu_torture_current ,
* incrementing the corresponding element of the pipeline array . The
* counter in the element should never be greater than 1 , otherwise , the
* RCU implementation is broken .
*/
2017-10-23 00:58:54 +00:00
static void rcu_torture_timer ( struct timer_list * unused )
2008-06-25 19:24:52 +00:00
{
2018-05-22 18:17:51 +00:00
atomic_long_inc ( & n_rcu_torture_timers ) ;
2018-05-22 18:59:31 +00:00
( void ) rcu_torture_one_read ( this_cpu_ptr ( & rcu_torture_timer_rand ) ) ;
2017-07-20 22:27:32 +00:00
/* Test call_rcu() invocation from interrupt handler. */
if ( cur_ops - > call ) {
struct rcu_head * rhp = kmalloc ( sizeof ( * rhp ) , GFP_NOWAIT ) ;
if ( rhp )
cur_ops - > call ( rhp , rcu_torture_timer_cb ) ;
}
2008-06-25 19:24:52 +00:00
}
2005-10-30 23:03:12 +00:00
/*
* RCU torture reader kthread . Repeatedly dereferences rcu_torture_current ,
* incrementing the corresponding element of the pipeline array . The
* counter in the element should never be greater than 1 , otherwise , the
* RCU implementation is broken .
*/
static int
rcu_torture_reader ( void * arg )
{
2014-01-27 19:49:39 +00:00
DEFINE_TORTURE_RANDOM ( rand ) ;
2008-06-25 19:24:52 +00:00
struct timer_list t ;
2005-10-30 23:03:12 +00:00
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_reader task started " ) ;
2014-02-11 07:34:49 +00:00
set_user_nice ( current , MAX_NICE ) ;
2009-06-25 16:08:17 +00:00
if ( irqreader & & cur_ops - > irq_capable )
2017-10-23 00:58:54 +00:00
timer_setup_on_stack ( & t , rcu_torture_timer , 0 ) ;
2005-11-14 00:07:22 +00:00
2005-10-30 23:03:12 +00:00
do {
2009-06-25 16:08:17 +00:00
if ( irqreader & & cur_ops - > irq_capable ) {
2008-06-25 19:24:52 +00:00
if ( ! timer_pending ( & t ) )
2010-02-23 01:05:04 +00:00
mod_timer ( & t , jiffies + 1 ) ;
2008-06-25 19:24:52 +00:00
}
2018-05-22 17:56:05 +00:00
if ( ! rcu_torture_one_read ( & rand ) )
2005-10-30 23:03:12 +00:00
schedule_timeout_interruptible ( HZ ) ;
2014-01-31 19:57:43 +00:00
stutter_wait ( " rcu_torture_reader " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2014-03-23 15:58:27 +00:00
if ( irqreader & & cur_ops - > irq_capable ) {
2008-06-25 19:24:52 +00:00
del_timer_sync ( & t ) ;
2014-03-23 15:58:27 +00:00
destroy_timer_on_stack ( & t ) ;
}
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_reader " ) ;
2005-10-30 23:03:12 +00:00
return 0 ;
}
/*
2014-07-14 13:16:15 +00:00
* Print torture statistics . Caller must ensure that there is only
* one call to this function at a given time ! ! ! This is normally
* accomplished by relying on the module system to only have one copy
* of the module loaded , and then by giving the rcu_torture_stats
* kthread full control ( or the init / cleanup functions when rcu_torture_stats
* thread is not running ) .
2005-10-30 23:03:12 +00:00
*/
rcu/torture: Dynamically allocate SRCU output buffer to avoid overflow
If the rcutorture SRCU output exceeds 4096 bytes, for example, if you
have more than about 75 CPUs, it will overflow the current statically
allocated buffer. This commit therefore replaces this static buffer
with a dynamically buffer whose size is based on the number of CPUs.
Benefits:
- Avoids both buffer overflow and output truncation.
- Handles an arbitrarily large number of CPUs.
- Straightforward implementation.
Shortcomings:
- Some memory is wasted:
1 cpu now comsumes 50 - 60 bytes, and this patch provides 200 bytes.
Therefore, for 1K CPUs, roughly 100KB of memory will be wasted.
However, the memory is freed immediately after printing, so this
wastage should not be a problem in practice.
Testing (Fedora16 2 CPUs, 2GB RAM x86_64):
- as module, with/without "torture_type=srcu".
- build-in not boot runnable, with/without "torture_type=srcu".
- build-in let boot runnable, with/without "torture_type=srcu".
Signed-off-by: Chen Gang <gang.chen@asianux.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
2013-11-07 02:30:25 +00:00
static void
2014-07-14 13:16:15 +00:00
rcu_torture_stats_print ( void )
2005-10-30 23:03:12 +00:00
{
int cpu ;
int i ;
long pipesummary [ RCU_TORTURE_PIPE_LEN + 1 ] = { 0 } ;
long batchsummary [ RCU_TORTURE_PIPE_LEN + 1 ] = { 0 } ;
2014-02-19 18:51:42 +00:00
static unsigned long rtcv_snap = ULONG_MAX ;
2017-08-30 17:40:17 +00:00
static bool splatted ;
2016-06-30 18:56:38 +00:00
struct task_struct * wtp ;
2005-10-30 23:03:12 +00:00
2006-03-28 09:56:37 +00:00
for_each_possible_cpu ( cpu ) {
2005-10-30 23:03:12 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + ) {
pipesummary [ i ] + = per_cpu ( rcu_torture_count , cpu ) [ i ] ;
batchsummary [ i ] + = per_cpu ( rcu_torture_batch , cpu ) [ i ] ;
}
}
for ( i = RCU_TORTURE_PIPE_LEN - 1 ; i > = 0 ; i - - ) {
if ( pipesummary [ i ] ! = 0 )
break ;
}
2014-07-14 13:16:15 +00:00
pr_alert ( " %s%s " , torture_type , TORTURE_FLAG ) ;
pr_cont ( " rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d " ,
rcu_torture_current ,
rcu_torture_current_version ,
list_empty ( & rcu_torture_freelist ) ,
atomic_read ( & n_rcu_torture_alloc ) ,
atomic_read ( & n_rcu_torture_alloc_fail ) ,
atomic_read ( & n_rcu_torture_free ) ) ;
2016-08-13 06:54:35 +00:00
pr_cont ( " rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld " ,
2014-07-14 13:16:15 +00:00
atomic_read ( & n_rcu_torture_mberror ) ,
2016-08-13 06:54:35 +00:00
n_rcu_torture_barrier_error ,
2014-07-14 13:16:15 +00:00
n_rcu_torture_boost_ktrerror ,
n_rcu_torture_boost_rterror ) ;
pr_cont ( " rtbf: %ld rtb: %ld nt: %ld " ,
n_rcu_torture_boost_failure ,
n_rcu_torture_boosts ,
2018-05-22 18:17:51 +00:00
atomic_long_read ( & n_rcu_torture_timers ) ) ;
2014-07-14 13:16:15 +00:00
torture_onoff_stats ( ) ;
2014-08-19 04:12:17 +00:00
pr_cont ( " barrier: %ld/%ld:%ld " ,
2014-07-14 13:16:15 +00:00
n_barrier_successes ,
n_barrier_attempts ,
n_rcu_torture_barrier_error ) ;
2014-08-19 04:12:17 +00:00
pr_cont ( " cbflood: %ld \n " , atomic_long_read ( & n_cbfloods ) ) ;
2014-07-14 13:16:15 +00:00
pr_alert ( " %s%s " , torture_type , TORTURE_FLAG ) ;
2010-09-02 23:16:14 +00:00
if ( atomic_read ( & n_rcu_torture_mberror ) ! = 0 | |
2012-02-21 01:51:45 +00:00
n_rcu_torture_barrier_error ! = 0 | |
2010-09-02 23:16:14 +00:00
n_rcu_torture_boost_ktrerror ! = 0 | |
n_rcu_torture_boost_rterror ! = 0 | |
2012-02-21 01:51:45 +00:00
n_rcu_torture_boost_failure ! = 0 | |
i > 1 ) {
2014-07-14 13:16:15 +00:00
pr_cont ( " %s " , " !!! " ) ;
2005-11-18 09:10:50 +00:00
atomic_inc ( & n_rcu_torture_error ) ;
2008-06-18 08:09:48 +00:00
WARN_ON_ONCE ( 1 ) ;
2005-11-18 09:10:50 +00:00
}
2014-07-14 13:16:15 +00:00
pr_cont ( " Reader Pipe: " ) ;
2005-10-30 23:03:12 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + )
2014-07-14 13:16:15 +00:00
pr_cont ( " %ld " , pipesummary [ i ] ) ;
pr_cont ( " \n " ) ;
pr_alert ( " %s%s " , torture_type , TORTURE_FLAG ) ;
pr_cont ( " Reader Batch: " ) ;
2006-06-27 09:54:03 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + )
2014-07-14 13:16:15 +00:00
pr_cont ( " %ld " , batchsummary [ i ] ) ;
pr_cont ( " \n " ) ;
pr_alert ( " %s%s " , torture_type , TORTURE_FLAG ) ;
pr_cont ( " Free-Block Circulation: " ) ;
2005-10-30 23:03:12 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + ) {
2014-07-14 13:16:15 +00:00
pr_cont ( " %d " , atomic_read ( & rcu_torture_wcount [ i ] ) ) ;
2005-10-30 23:03:12 +00:00
}
2014-07-14 13:16:15 +00:00
pr_cont ( " \n " ) ;
2007-05-08 07:33:20 +00:00
if ( cur_ops - > stats )
2014-07-14 13:16:15 +00:00
cur_ops - > stats ( ) ;
2014-02-19 18:51:42 +00:00
if ( rtcv_snap = = rcu_torture_current_version & &
rcu_torture_current ! = NULL ) {
2017-04-19 00:17:35 +00:00
int __maybe_unused flags = 0 ;
2018-05-01 13:42:51 +00:00
unsigned long __maybe_unused gp_seq = 0 ;
2014-02-19 18:51:42 +00:00
rcutorture_get_gp_data ( cur_ops - > ttype ,
2018-05-01 13:42:51 +00:00
& flags , & gp_seq ) ;
2017-04-19 00:17:35 +00:00
srcutorture_get_gp_data ( cur_ops - > ttype , srcu_ctlp ,
2018-05-01 13:42:51 +00:00
& flags , & gp_seq ) ;
2016-06-30 18:56:38 +00:00
wtp = READ_ONCE ( writer_task ) ;
2018-05-01 13:42:51 +00:00
pr_alert ( " ??? Writer stall state %s(%d) g%lu f%#x ->state %#lx cpu %d \n " ,
2015-11-17 21:35:28 +00:00
rcu_torture_writer_state_getname ( ) ,
2018-05-01 13:42:51 +00:00
rcu_torture_writer_state , gp_seq , flags ,
2017-06-19 17:03:22 +00:00
wtp = = NULL ? ~ 0UL : wtp - > state ,
wtp = = NULL ? - 1 : ( int ) task_cpu ( wtp ) ) ;
2017-08-30 17:40:17 +00:00
if ( ! splatted & & wtp ) {
sched_show_task ( wtp ) ;
splatted = true ;
}
2014-03-12 14:10:41 +00:00
show_rcu_gp_kthreads ( ) ;
2016-03-22 02:46:04 +00:00
rcu_ftrace_dump ( DUMP_ALL ) ;
2014-02-19 18:51:42 +00:00
}
rtcv_snap = rcu_torture_current_version ;
2005-10-30 23:03:12 +00:00
}
/*
* Periodically prints torture statistics , if periodic statistics printing
* was specified via the stat_interval module parameter .
*/
static int
rcu_torture_stats ( void * arg )
{
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_stats task started " ) ;
2005-10-30 23:03:12 +00:00
do {
schedule_timeout_interruptible ( stat_interval * HZ ) ;
rcu_torture_stats_print ( ) ;
2014-01-29 15:40:27 +00:00
torture_shutdown_absorb ( " rcu_torture_stats " ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_stats " ) ;
2008-06-18 12:21:44 +00:00
return 0 ;
}
2018-05-17 18:33:17 +00:00
static void
2013-07-12 20:50:28 +00:00
rcu_torture_print_module_parms ( struct rcu_torture_ops * cur_ops , const char * tag )
2006-03-24 11:15:58 +00:00
{
2012-08-09 23:30:45 +00:00
pr_alert ( " %s " TORTURE_FLAG
" --- %s: nreaders=%d nfakewriters=%d "
" stat_interval=%d verbose=%d test_no_idle_hz=%d "
" shuffle_interval=%d stutter=%d irqreader=%d "
" fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
" test_boost=%d/%d test_boost_interval=%d "
" test_boost_duration=%d shutdown_secs=%d "
2017-08-18 23:11:37 +00:00
" stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
2012-10-20 19:56:06 +00:00
" n_barrier_cbs=%d "
2012-08-09 23:30:45 +00:00
" onoff_interval=%d onoff_holdoff=%d \n " ,
torture_type , tag , nrealreaders , nfakewriters ,
stat_interval , verbose , test_no_idle_hz , shuffle_interval ,
stutter , irqreader , fqs_duration , fqs_holdoff , fqs_stutter ,
test_boost , cur_ops - > can_boost ,
test_boost_interval , test_boost_duration , shutdown_secs ,
2017-08-18 23:11:37 +00:00
stall_cpu , stall_cpu_holdoff , stall_cpu_irqsoff ,
2012-10-20 19:56:06 +00:00
n_barrier_cbs ,
2012-08-09 23:30:45 +00:00
onoff_interval , onoff_holdoff ) ;
2006-03-24 11:15:58 +00:00
}
2016-08-18 12:57:22 +00:00
static int rcutorture_booster_cleanup ( unsigned int cpu )
2010-09-02 23:16:14 +00:00
{
struct task_struct * t ;
if ( boost_tasks [ cpu ] = = NULL )
2016-08-18 12:57:22 +00:00
return 0 ;
2010-09-02 23:16:14 +00:00
mutex_lock ( & boost_mutex ) ;
t = boost_tasks [ cpu ] ;
boost_tasks [ cpu ] = NULL ;
2018-06-10 23:45:43 +00:00
rcu_torture_enable_rt_throttle ( ) ;
2010-09-02 23:16:14 +00:00
mutex_unlock ( & boost_mutex ) ;
/* This must be outside of the mutex, otherwise deadlock! */
2014-02-04 19:47:08 +00:00
torture_stop_kthread ( rcu_torture_boost , t ) ;
2016-08-18 12:57:22 +00:00
return 0 ;
2010-09-02 23:16:14 +00:00
}
2016-08-18 12:57:22 +00:00
static int rcutorture_booster_init ( unsigned int cpu )
2010-09-02 23:16:14 +00:00
{
int retval ;
if ( boost_tasks [ cpu ] ! = NULL )
return 0 ; /* Already created, nothing more to do. */
/* Don't allow time recalculation while creating a new task. */
mutex_lock ( & boost_mutex ) ;
2018-06-10 23:45:43 +00:00
rcu_torture_disable_rt_throttle ( ) ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " Creating rcu_torture_boost task " ) ;
2011-06-16 22:53:18 +00:00
boost_tasks [ cpu ] = kthread_create_on_node ( rcu_torture_boost , NULL ,
cpu_to_node ( cpu ) ,
" rcu_torture_boost " ) ;
2010-09-02 23:16:14 +00:00
if ( IS_ERR ( boost_tasks [ cpu ] ) ) {
retval = PTR_ERR ( boost_tasks [ cpu ] ) ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_boost task create failed " ) ;
2010-09-02 23:16:14 +00:00
n_rcu_torture_boost_ktrerror + + ;
boost_tasks [ cpu ] = NULL ;
mutex_unlock ( & boost_mutex ) ;
return retval ;
}
kthread_bind ( boost_tasks [ cpu ] , cpu ) ;
wake_up_process ( boost_tasks [ cpu ] ) ;
mutex_unlock ( & boost_mutex ) ;
return 0 ;
}
2012-01-20 23:36:33 +00:00
/*
* CPU - stall kthread . It waits as specified by stall_cpu_holdoff , then
* induces a CPU stall for the time specified by stall_cpu .
*/
2013-06-19 18:52:21 +00:00
static int rcu_torture_stall ( void * args )
2012-01-20 23:36:33 +00:00
{
unsigned long stop_at ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_stall task started " ) ;
2012-01-20 23:36:33 +00:00
if ( stall_cpu_holdoff > 0 ) {
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_stall begin holdoff " ) ;
2012-01-20 23:36:33 +00:00
schedule_timeout_interruptible ( stall_cpu_holdoff * HZ ) ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_stall end holdoff " ) ;
2012-01-20 23:36:33 +00:00
}
if ( ! kthread_should_stop ( ) ) {
2018-06-18 14:47:34 +00:00
stop_at = ktime_get_seconds ( ) + stall_cpu ;
2012-01-20 23:36:33 +00:00
/* RCU CPU stall is expected behavior in following code. */
rcu_read_lock ( ) ;
2017-08-18 23:11:37 +00:00
if ( stall_cpu_irqsoff )
local_irq_disable ( ) ;
else
preempt_disable ( ) ;
pr_alert ( " rcu_torture_stall start on CPU %d. \n " ,
smp_processor_id ( ) ) ;
2018-06-18 14:47:34 +00:00
while ( ULONG_CMP_LT ( ( unsigned long ) ktime_get_seconds ( ) ,
stop_at ) )
2012-01-20 23:36:33 +00:00
continue ; /* Induce RCU CPU stall warning. */
2017-08-18 23:11:37 +00:00
if ( stall_cpu_irqsoff )
local_irq_enable ( ) ;
else
preempt_enable ( ) ;
2012-01-20 23:36:33 +00:00
rcu_read_unlock ( ) ;
2012-08-09 23:30:45 +00:00
pr_alert ( " rcu_torture_stall end. \n " ) ;
2012-01-20 23:36:33 +00:00
}
2014-01-29 15:40:27 +00:00
torture_shutdown_absorb ( " rcu_torture_stall " ) ;
2012-01-20 23:36:33 +00:00
while ( ! kthread_should_stop ( ) )
schedule_timeout_interruptible ( 10 * HZ ) ;
return 0 ;
}
/* Spawn CPU-stall kthread, if stall_cpu specified. */
static int __init rcu_torture_stall_init ( void )
{
if ( stall_cpu < = 0 )
return 0 ;
2014-02-03 19:52:27 +00:00
return torture_create_kthread ( rcu_torture_stall , NULL , stall_task ) ;
2012-01-20 23:36:33 +00:00
}
2012-02-21 01:51:45 +00:00
/* Callback function for RCU barrier testing. */
2014-02-27 11:46:57 +00:00
static void rcu_torture_barrier_cbf ( struct rcu_head * rcu )
2012-02-21 01:51:45 +00:00
{
atomic_inc ( & barrier_cbs_invoked ) ;
}
/* kthread function to register callbacks used to test RCU barriers. */
static int rcu_torture_barrier_cbs ( void * arg )
{
long myid = ( long ) arg ;
2012-05-29 02:21:41 +00:00
bool lastphase = 0 ;
2013-09-24 22:04:06 +00:00
bool newphase ;
2012-02-21 01:51:45 +00:00
struct rcu_head rcu ;
init_rcu_head_on_stack ( & rcu ) ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_barrier_cbs task started " ) ;
2014-02-11 07:34:49 +00:00
set_user_nice ( current , MAX_NICE ) ;
2012-02-21 01:51:45 +00:00
do {
wait_event ( barrier_cbs_wq [ myid ] ,
2013-09-24 22:04:06 +00:00
( newphase =
2015-04-13 18:58:08 +00:00
smp_load_acquire ( & barrier_phase ) ) ! = lastphase | |
2014-01-30 23:49:29 +00:00
torture_must_stop ( ) ) ;
2013-09-24 22:04:06 +00:00
lastphase = newphase ;
2014-01-30 23:49:29 +00:00
if ( torture_must_stop ( ) )
2012-02-21 01:51:45 +00:00
break ;
2015-04-13 18:58:08 +00:00
/*
* The above smp_load_acquire ( ) ensures barrier_phase load
2016-05-02 19:20:51 +00:00
* is ordered before the following - > call ( ) .
2015-04-13 18:58:08 +00:00
*/
2016-03-30 18:40:44 +00:00
local_irq_disable ( ) ; /* Just to test no-irq call_rcu(). */
2012-02-21 01:51:45 +00:00
cur_ops - > call ( & rcu , rcu_torture_barrier_cbf ) ;
2016-03-30 18:40:44 +00:00
local_irq_enable ( ) ;
2012-02-21 01:51:45 +00:00
if ( atomic_dec_and_test ( & barrier_cbs_count ) )
wake_up ( & barrier_wq ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2014-07-01 18:59:36 +00:00
if ( cur_ops - > cb_barrier ! = NULL )
cur_ops - > cb_barrier ( ) ;
2012-02-21 01:51:45 +00:00
destroy_rcu_head_on_stack ( & rcu ) ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_barrier_cbs " ) ;
2012-02-21 01:51:45 +00:00
return 0 ;
}
/* kthread function to drive and coordinate RCU barrier testing. */
static int rcu_torture_barrier ( void * arg )
{
int i ;
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_STRING ( " rcu_torture_barrier task starting " ) ;
2012-02-21 01:51:45 +00:00
do {
atomic_set ( & barrier_cbs_invoked , 0 ) ;
atomic_set ( & barrier_cbs_count , n_barrier_cbs ) ;
2015-04-13 18:58:08 +00:00
/* Ensure barrier_phase ordered after prior assignments. */
smp_store_release ( & barrier_phase , ! barrier_phase ) ;
2012-02-21 01:51:45 +00:00
for ( i = 0 ; i < n_barrier_cbs ; i + + )
wake_up ( & barrier_cbs_wq [ i ] ) ;
wait_event ( barrier_wq ,
atomic_read ( & barrier_cbs_count ) = = 0 | |
2014-01-30 23:49:29 +00:00
torture_must_stop ( ) ) ;
if ( torture_must_stop ( ) )
2012-02-21 01:51:45 +00:00
break ;
n_barrier_attempts + + ;
2013-09-24 22:04:06 +00:00
cur_ops - > cb_barrier ( ) ; /* Implies smp_mb() for wait_event(). */
2012-02-21 01:51:45 +00:00
if ( atomic_read ( & barrier_cbs_invoked ) ! = n_barrier_cbs ) {
n_rcu_torture_barrier_error + + ;
2014-12-18 02:39:54 +00:00
pr_err ( " barrier_cbs_invoked = %d, n_barrier_cbs = %d \n " ,
atomic_read ( & barrier_cbs_invoked ) ,
n_barrier_cbs ) ;
2012-02-21 01:51:45 +00:00
WARN_ON_ONCE ( 1 ) ;
2018-06-19 22:14:19 +00:00
} else {
n_barrier_successes + + ;
2012-02-21 01:51:45 +00:00
}
schedule_timeout_interruptible ( HZ / 10 ) ;
2014-01-30 23:49:29 +00:00
} while ( ! torture_must_stop ( ) ) ;
2014-02-01 01:37:28 +00:00
torture_kthread_stopping ( " rcu_torture_barrier " ) ;
2012-02-21 01:51:45 +00:00
return 0 ;
}
/* Initialize RCU barrier testing. */
static int rcu_torture_barrier_init ( void )
{
int i ;
int ret ;
2015-05-14 22:35:43 +00:00
if ( n_barrier_cbs < = 0 )
2012-02-21 01:51:45 +00:00
return 0 ;
if ( cur_ops - > call = = NULL | | cur_ops - > cb_barrier = = NULL ) {
2012-08-09 23:30:45 +00:00
pr_alert ( " %s " TORTURE_FLAG
" Call or barrier ops missing for %s, \n " ,
torture_type , cur_ops - > name ) ;
pr_alert ( " %s " TORTURE_FLAG
" RCU barrier testing omitted from run. \n " ,
torture_type ) ;
2012-02-21 01:51:45 +00:00
return 0 ;
}
atomic_set ( & barrier_cbs_count , 0 ) ;
atomic_set ( & barrier_cbs_invoked , 0 ) ;
barrier_cbs_tasks =
2017-12-01 22:26:56 +00:00
kcalloc ( n_barrier_cbs , sizeof ( barrier_cbs_tasks [ 0 ] ) ,
2012-02-21 01:51:45 +00:00
GFP_KERNEL ) ;
barrier_cbs_wq =
2017-12-01 22:26:56 +00:00
kcalloc ( n_barrier_cbs , sizeof ( barrier_cbs_wq [ 0 ] ) , GFP_KERNEL ) ;
2012-12-20 19:11:28 +00:00
if ( barrier_cbs_tasks = = NULL | | ! barrier_cbs_wq )
2012-02-21 01:51:45 +00:00
return - ENOMEM ;
for ( i = 0 ; i < n_barrier_cbs ; i + + ) {
init_waitqueue_head ( & barrier_cbs_wq [ i ] ) ;
2014-02-03 19:52:27 +00:00
ret = torture_create_kthread ( rcu_torture_barrier_cbs ,
( void * ) ( long ) i ,
barrier_cbs_tasks [ i ] ) ;
if ( ret )
2012-02-21 01:51:45 +00:00
return ret ;
}
2014-02-03 19:52:27 +00:00
return torture_create_kthread ( rcu_torture_barrier , NULL , barrier_task ) ;
2012-02-21 01:51:45 +00:00
}
/* Clean up after RCU barrier testing. */
static void rcu_torture_barrier_cleanup ( void )
{
int i ;
2014-02-04 19:47:08 +00:00
torture_stop_kthread ( rcu_torture_barrier , barrier_task ) ;
2012-02-21 01:51:45 +00:00
if ( barrier_cbs_tasks ! = NULL ) {
2014-02-04 19:47:08 +00:00
for ( i = 0 ; i < n_barrier_cbs ; i + + )
torture_stop_kthread ( rcu_torture_barrier_cbs ,
barrier_cbs_tasks [ i ] ) ;
2012-02-21 01:51:45 +00:00
kfree ( barrier_cbs_tasks ) ;
barrier_cbs_tasks = NULL ;
}
if ( barrier_cbs_wq ! = NULL ) {
kfree ( barrier_cbs_wq ) ;
barrier_cbs_wq = NULL ;
}
}
2018-06-19 22:14:18 +00:00
static bool rcu_torture_can_boost ( void )
{
static int boost_warn_once ;
int prio ;
if ( ! ( test_boost = = 1 & & cur_ops - > can_boost ) & & test_boost ! = 2 )
return false ;
prio = rcu_get_gp_kthreads_prio ( ) ;
if ( ! prio )
return false ;
if ( prio < 2 ) {
if ( boost_warn_once = = 1 )
return false ;
2018-06-19 22:14:19 +00:00
pr_alert ( " %s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line. \n " , KBUILD_MODNAME ) ;
2018-06-19 22:14:18 +00:00
boost_warn_once = 1 ;
return false ;
}
return true ;
}
2016-08-18 12:57:22 +00:00
static enum cpuhp_state rcutor_hp ;
2010-09-02 23:16:14 +00:00
2005-10-30 23:03:12 +00:00
static void
rcu_torture_cleanup ( void )
{
2018-04-19 15:43:11 +00:00
int flags = 0 ;
2018-05-01 13:42:51 +00:00
unsigned long gp_seq = 0 ;
2005-10-30 23:03:12 +00:00
int i ;
2014-09-12 03:40:21 +00:00
if ( torture_cleanup_begin ( ) ) {
2008-12-16 00:13:07 +00:00
if ( cur_ops - > cb_barrier ! = NULL )
cur_ops - > cb_barrier ( ) ;
return ;
}
2006-01-08 09:03:42 +00:00
2012-02-21 01:51:45 +00:00
rcu_torture_barrier_cleanup ( ) ;
2014-02-04 19:47:08 +00:00
torture_stop_kthread ( rcu_torture_stall , stall_task ) ;
torture_stop_kthread ( rcu_torture_writer , writer_task ) ;
2005-10-30 23:03:12 +00:00
2007-05-08 07:33:20 +00:00
if ( reader_tasks ) {
2014-02-04 19:47:08 +00:00
for ( i = 0 ; i < nrealreaders ; i + + )
torture_stop_kthread ( rcu_torture_reader ,
reader_tasks [ i ] ) ;
2005-10-30 23:03:12 +00:00
kfree ( reader_tasks ) ;
}
rcu_torture_current = NULL ;
2007-05-08 07:33:20 +00:00
if ( fakewriter_tasks ) {
2006-10-04 09:17:13 +00:00
for ( i = 0 ; i < nfakewriters ; i + + ) {
2014-02-04 19:47:08 +00:00
torture_stop_kthread ( rcu_torture_fakewriter ,
fakewriter_tasks [ i ] ) ;
2006-10-04 09:17:13 +00:00
}
kfree ( fakewriter_tasks ) ;
fakewriter_tasks = NULL ;
}
2018-05-01 13:42:51 +00:00
rcutorture_get_gp_data ( cur_ops - > ttype , & flags , & gp_seq ) ;
srcutorture_get_gp_data ( cur_ops - > ttype , srcu_ctlp , & flags , & gp_seq ) ;
pr_alert ( " %s: End-test grace-period state: g%lu f%#x \n " ,
cur_ops - > name , gp_seq , flags ) ;
2014-02-04 19:47:08 +00:00
torture_stop_kthread ( rcu_torture_stats , stats_task ) ;
torture_stop_kthread ( rcu_torture_fqs , fqs_task ) ;
2014-08-19 04:12:17 +00:00
for ( i = 0 ; i < ncbflooders ; i + + )
torture_stop_kthread ( rcu_torture_cbflood , cbflood_task [ i ] ) ;
2018-06-19 22:14:18 +00:00
if ( rcu_torture_can_boost ( ) )
2016-08-18 12:57:22 +00:00
cpuhp_remove_state ( rcutor_hp ) ;
2010-01-04 23:09:10 +00:00
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
/*
* Wait for all RCU callbacks to fire , then do flavor - specific
* cleanup operations .
*/
2008-05-12 19:21:05 +00:00
if ( cur_ops - > cb_barrier ! = NULL )
cur_ops - > cb_barrier ( ) ;
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
if ( cur_ops - > cleanup ! = NULL )
cur_ops - > cleanup ( ) ;
2005-10-30 23:03:12 +00:00
rcu_torture_stats_print ( ) ; /* -After- the stats thread is stopped! */
2006-06-27 09:54:03 +00:00
2012-02-21 01:51:45 +00:00
if ( atomic_read ( & n_rcu_torture_error ) | | n_rcu_torture_barrier_error )
2010-09-02 23:16:14 +00:00
rcu_torture_print_module_parms ( cur_ops , " End of test: FAILURE " ) ;
2014-01-28 23:58:22 +00:00
else if ( torture_onoff_failures ( ) )
2012-01-10 20:51:14 +00:00
rcu_torture_print_module_parms ( cur_ops ,
" End of test: RCU_HOTPLUG " ) ;
2006-03-24 11:15:58 +00:00
else
2010-09-02 23:16:14 +00:00
rcu_torture_print_module_parms ( cur_ops , " End of test: SUCCESS " ) ;
2014-09-12 03:40:21 +00:00
torture_cleanup_end ( ) ;
2005-10-30 23:03:12 +00:00
}
2013-04-24 00:05:42 +00:00
# ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
static void rcu_torture_leak_cb ( struct rcu_head * rhp )
{
}
static void rcu_torture_err_cb ( struct rcu_head * rhp )
{
/*
* This - might - happen due to race conditions , but is unlikely .
* The scenario that leads to this happening is that the
* first of the pair of duplicate callbacks is queued ,
* someone else starts a grace period that includes that
* callback , then the second of the pair must wait for the
* next grace period . Unlikely , but can happen . If it
* does happen , the debug - objects subsystem won ' t have splatted .
*/
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: duplicated callback was invoked. \n " , KBUILD_MODNAME ) ;
2013-04-24 00:05:42 +00:00
}
# endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
/*
* Verify that double - free causes debug - objects to complain , but only
* if CONFIG_DEBUG_OBJECTS_RCU_HEAD = y . Otherwise , say that the test
* cannot be carried out .
*/
static void rcu_test_debug_objects ( void )
{
# ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
struct rcu_head rh1 ;
struct rcu_head rh2 ;
init_rcu_head_on_stack ( & rh1 ) ;
init_rcu_head_on_stack ( & rh2 ) ;
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: WARN: Duplicate call_rcu() test starting. \n " , KBUILD_MODNAME ) ;
2013-04-24 00:05:42 +00:00
/* Try to queue the rh2 pair of callbacks for the same grace period. */
preempt_disable ( ) ; /* Prevent preemption from interrupting test. */
rcu_read_lock ( ) ; /* Make it impossible to finish a grace period. */
call_rcu ( & rh1 , rcu_torture_leak_cb ) ; /* Start grace period. */
local_irq_disable ( ) ; /* Make it harder to start a new grace period. */
call_rcu ( & rh2 , rcu_torture_leak_cb ) ;
call_rcu ( & rh2 , rcu_torture_err_cb ) ; /* Duplicate callback. */
local_irq_enable ( ) ;
rcu_read_unlock ( ) ;
preempt_enable ( ) ;
/* Wait for them all to get done so we can safely return. */
rcu_barrier ( ) ;
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: WARN: Duplicate call_rcu() test complete. \n " , KBUILD_MODNAME ) ;
2013-04-24 00:05:42 +00:00
destroy_rcu_head_on_stack ( & rh1 ) ;
destroy_rcu_head_on_stack ( & rh2 ) ;
# else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
2017-12-01 23:22:38 +00:00
pr_alert ( " %s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu() \n " , KBUILD_MODNAME ) ;
2013-04-24 00:05:42 +00:00
# endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
}
2007-05-08 07:25:24 +00:00
static int __init
2005-10-30 23:03:12 +00:00
rcu_torture_init ( void )
{
int i ;
int cpu ;
int firsterr = 0 ;
2013-06-12 22:12:21 +00:00
static struct rcu_torture_ops * torture_ops [ ] = {
rcutorture: Test SRCU cleanup code path
The current rcutorture testing does not do any cleanup operations.
This works because the srcu_struct is statically allocated, but it
does represent a memory leak of the associated dynamically allocated
->per_cpu_ref per-CPU variables. However, rcutorture currently uses
a statically allocated srcu_struct, which cannot legally be passed to
cleanup_srcu_struct(). Therefore, this commit adds a second form
of srcu (called srcud) that dynamically allocates and frees the
associated per-CPU variables. This commit also adds a ->cleanup()
member to rcu_torture_ops that is invoked at the end of the test,
after ->cb_barriers(). This ->cleanup() pointer is NULL for all
existing tests, and thus only used for scrud. Finally, the SRCU-P
torture-test configuration selects scrud instead of srcu, with SRCU-N
continuing to use srcu, thereby testing both static and dynamic
srcu_struct structures.
Reported-by: "Ahmed, Iftekhar" <ahmedi@onid.oregonstate.edu>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2015-04-14 19:28:22 +00:00
& rcu_ops , & rcu_bh_ops , & rcu_busted_ops , & srcu_ops , & srcud_ops ,
2018-05-25 14:29:25 +00:00
& busted_srcud_ops , & sched_ops , & tasks_ops ,
2013-06-12 22:12:21 +00:00
} ;
2005-10-30 23:03:12 +00:00
2017-11-22 04:19:17 +00:00
if ( ! torture_init_begin ( torture_type , verbose ) )
2014-04-07 16:14:11 +00:00
return - EBUSY ;
2008-12-16 00:13:07 +00:00
2005-10-30 23:03:12 +00:00
/* Process args and tell the world that the torturer is on the job. */
2007-05-08 07:33:22 +00:00
for ( i = 0 ; i < ARRAY_SIZE ( torture_ops ) ; i + + ) {
2006-06-27 09:54:03 +00:00
cur_ops = torture_ops [ i ] ;
2007-05-08 07:33:22 +00:00
if ( strcmp ( torture_type , cur_ops - > name ) = = 0 )
2006-06-27 09:54:03 +00:00
break ;
}
2007-05-08 07:33:22 +00:00
if ( i = = ARRAY_SIZE ( torture_ops ) ) {
2012-08-09 23:30:45 +00:00
pr_alert ( " rcu-torture: invalid torture type: \" %s \" \n " ,
torture_type ) ;
pr_alert ( " rcu-torture types: " ) ;
2009-10-26 02:03:54 +00:00
for ( i = 0 ; i < ARRAY_SIZE ( torture_ops ) ; i + + )
2018-05-14 20:27:33 +00:00
pr_cont ( " %s " , torture_ops [ i ] - > name ) ;
pr_cont ( " \n " ) ;
2015-08-24 18:37:58 +00:00
firsterr = - EINVAL ;
goto unwind ;
2006-06-27 09:54:03 +00:00
}
2010-01-04 23:09:10 +00:00
if ( cur_ops - > fqs = = NULL & & fqs_duration ! = 0 ) {
2012-08-09 23:30:45 +00:00
pr_alert ( " rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled. \n " ) ;
2010-01-04 23:09:10 +00:00
fqs_duration = 0 ;
}
2007-05-08 07:33:20 +00:00
if ( cur_ops - > init )
2015-08-24 18:37:58 +00:00
cur_ops - > init ( ) ;
2006-06-27 09:54:03 +00:00
2014-03-12 17:26:35 +00:00
if ( nreaders > = 0 ) {
2005-10-30 23:03:12 +00:00
nrealreaders = nreaders ;
2014-03-12 17:26:35 +00:00
} else {
2015-03-12 20:55:48 +00:00
nrealreaders = num_online_cpus ( ) - 2 - nreaders ;
2014-03-12 17:26:35 +00:00
if ( nrealreaders < = 0 )
nrealreaders = 1 ;
}
2010-09-02 23:16:14 +00:00
rcu_torture_print_module_parms ( cur_ops , " Start of test " ) ;
2005-10-30 23:03:12 +00:00
/* Set up the freelist. */
INIT_LIST_HEAD ( & rcu_torture_freelist ) ;
2007-05-08 07:33:14 +00:00
for ( i = 0 ; i < ARRAY_SIZE ( rcu_tortures ) ; i + + ) {
2005-11-18 09:10:50 +00:00
rcu_tortures [ i ] . rtort_mbtest = 0 ;
2005-10-30 23:03:12 +00:00
list_add_tail ( & rcu_tortures [ i ] . rtort_free ,
& rcu_torture_freelist ) ;
}
/* Initialize the statistics so that each run gets its own numbers. */
rcu_torture_current = NULL ;
rcu_torture_current_version = 0 ;
atomic_set ( & n_rcu_torture_alloc , 0 ) ;
atomic_set ( & n_rcu_torture_alloc_fail , 0 ) ;
atomic_set ( & n_rcu_torture_free , 0 ) ;
2005-11-18 09:10:50 +00:00
atomic_set ( & n_rcu_torture_mberror , 0 ) ;
atomic_set ( & n_rcu_torture_error , 0 ) ;
2012-02-21 01:51:45 +00:00
n_rcu_torture_barrier_error = 0 ;
2010-09-02 23:16:14 +00:00
n_rcu_torture_boost_ktrerror = 0 ;
n_rcu_torture_boost_rterror = 0 ;
n_rcu_torture_boost_failure = 0 ;
n_rcu_torture_boosts = 0 ;
2005-10-30 23:03:12 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + )
atomic_set ( & rcu_torture_wcount [ i ] , 0 ) ;
2006-03-28 09:56:37 +00:00
for_each_possible_cpu ( cpu ) {
2005-10-30 23:03:12 +00:00
for ( i = 0 ; i < RCU_TORTURE_PIPE_LEN + 1 ; i + + ) {
per_cpu ( rcu_torture_count , cpu ) [ i ] = 0 ;
per_cpu ( rcu_torture_batch , cpu ) [ i ] = 0 ;
}
}
/* Start up the kthreads. */
2014-02-03 19:52:27 +00:00
firsterr = torture_create_kthread ( rcu_torture_writer , NULL ,
writer_task ) ;
if ( firsterr )
2005-10-30 23:03:12 +00:00
goto unwind ;
2015-05-14 22:42:40 +00:00
if ( nfakewriters > 0 ) {
2017-12-01 22:26:56 +00:00
fakewriter_tasks = kcalloc ( nfakewriters ,
2015-05-14 22:42:40 +00:00
sizeof ( fakewriter_tasks [ 0 ] ) ,
GFP_KERNEL ) ;
if ( fakewriter_tasks = = NULL ) {
VERBOSE_TOROUT_ERRSTRING ( " out of memory " ) ;
firsterr = - ENOMEM ;
goto unwind ;
}
2006-10-04 09:17:13 +00:00
}
for ( i = 0 ; i < nfakewriters ; i + + ) {
2014-02-03 19:52:27 +00:00
firsterr = torture_create_kthread ( rcu_torture_fakewriter ,
NULL , fakewriter_tasks [ i ] ) ;
if ( firsterr )
2006-10-04 09:17:13 +00:00
goto unwind ;
}
2017-12-01 22:26:56 +00:00
reader_tasks = kcalloc ( nrealreaders , sizeof ( reader_tasks [ 0 ] ) ,
2005-10-30 23:03:12 +00:00
GFP_KERNEL ) ;
if ( reader_tasks = = NULL ) {
2014-01-29 15:25:25 +00:00
VERBOSE_TOROUT_ERRSTRING ( " out of memory " ) ;
2005-10-30 23:03:12 +00:00
firsterr = - ENOMEM ;
goto unwind ;
}
for ( i = 0 ; i < nrealreaders ; i + + ) {
2014-02-03 19:52:27 +00:00
firsterr = torture_create_kthread ( rcu_torture_reader , NULL ,
reader_tasks [ i ] ) ;
if ( firsterr )
2005-10-30 23:03:12 +00:00
goto unwind ;
}
if ( stat_interval > 0 ) {
2014-02-03 19:52:27 +00:00
firsterr = torture_create_kthread ( rcu_torture_stats , NULL ,
stats_task ) ;
if ( firsterr )
2005-10-30 23:03:12 +00:00
goto unwind ;
}
2015-05-14 23:55:45 +00:00
if ( test_no_idle_hz & & shuffle_interval > 0 ) {
2014-01-28 23:29:21 +00:00
firsterr = torture_shuffle_init ( shuffle_interval * HZ ) ;
if ( firsterr )
2006-01-08 09:03:42 +00:00
goto unwind ;
}
2008-06-18 12:21:44 +00:00
if ( stutter < 0 )
stutter = 0 ;
if ( stutter ) {
2014-01-31 19:57:43 +00:00
firsterr = torture_stutter_init ( stutter * HZ ) ;
if ( firsterr )
2008-06-18 12:21:44 +00:00
goto unwind ;
}
2010-01-04 23:09:10 +00:00
if ( fqs_duration < 0 )
fqs_duration = 0 ;
if ( fqs_duration ) {
2014-01-31 19:57:43 +00:00
/* Create the fqs thread */
2014-03-18 03:56:45 +00:00
firsterr = torture_create_kthread ( rcu_torture_fqs , NULL ,
fqs_task ) ;
2014-02-03 19:52:27 +00:00
if ( firsterr )
2010-01-04 23:09:10 +00:00
goto unwind ;
}
2010-09-02 23:16:14 +00:00
if ( test_boost_interval < 1 )
test_boost_interval = 1 ;
if ( test_boost_duration < 2 )
test_boost_duration = 2 ;
2018-06-19 22:14:18 +00:00
if ( rcu_torture_can_boost ( ) ) {
2010-09-02 23:16:14 +00:00
boost_starttime = jiffies + test_boost_interval * HZ ;
2016-08-18 12:57:22 +00:00
firsterr = cpuhp_setup_state ( CPUHP_AP_ONLINE_DYN , " RCU_TORTURE " ,
rcutorture_booster_init ,
rcutorture_booster_cleanup ) ;
if ( firsterr < 0 )
goto unwind ;
rcutor_hp = firsterr ;
2010-09-02 23:16:14 +00:00
}
2014-01-31 23:15:02 +00:00
firsterr = torture_shutdown_init ( shutdown_secs , rcu_torture_cleanup ) ;
if ( firsterr )
2012-02-18 06:12:18 +00:00
goto unwind ;
2018-05-08 16:20:34 +00:00
firsterr = torture_onoff_init ( onoff_holdoff * HZ , onoff_interval ) ;
2014-01-31 23:15:02 +00:00
if ( firsterr )
2012-02-18 06:12:18 +00:00
goto unwind ;
2014-01-31 23:15:02 +00:00
firsterr = rcu_torture_stall_init ( ) ;
if ( firsterr )
2012-02-18 06:12:18 +00:00
goto unwind ;
2014-01-31 23:15:02 +00:00
firsterr = rcu_torture_barrier_init ( ) ;
if ( firsterr )
2012-02-21 01:51:45 +00:00
goto unwind ;
2013-04-24 00:05:42 +00:00
if ( object_debug )
rcu_test_debug_objects ( ) ;
2014-08-19 04:12:17 +00:00
if ( cbflood_n_burst > 0 ) {
/* Create the cbflood threads */
ncbflooders = ( num_online_cpus ( ) + 3 ) / 4 ;
cbflood_task = kcalloc ( ncbflooders , sizeof ( * cbflood_task ) ,
GFP_KERNEL ) ;
if ( ! cbflood_task ) {
VERBOSE_TOROUT_ERRSTRING ( " out of memory " ) ;
firsterr = - ENOMEM ;
goto unwind ;
}
for ( i = 0 ; i < ncbflooders ; i + + ) {
firsterr = torture_create_kthread ( rcu_torture_cbflood ,
NULL ,
cbflood_task [ i ] ) ;
if ( firsterr )
goto unwind ;
}
}
2014-01-30 21:38:09 +00:00
torture_init_end ( ) ;
2005-10-30 23:03:12 +00:00
return 0 ;
unwind :
2014-01-30 21:38:09 +00:00
torture_init_end ( ) ;
2005-10-30 23:03:12 +00:00
rcu_torture_cleanup ( ) ;
return firsterr ;
}
module_init ( rcu_torture_init ) ;
module_exit ( rcu_torture_cleanup ) ;