mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-27 12:57:53 +00:00
Code Issues Releases Wiki Activity

percpu_ref: replace pcpu_ prefix with percpu_

Browse source 
percpu_ref uses pcpu_ prefix for internal stuff and percpu_ for
externally visible ones.  This is the same convention used in the
percpu allocator implementation.  It works fine there but percpu_ref
doesn't have too much internal-only stuff and scattered usages of
pcpu_ prefix are confusing than helpful.

This patch replaces all pcpu_ prefixes with percpu_.  This is pure
rename and there's no functional change.  Note that PCPU_REF_DEAD is
renamed to __PERCPU_REF_DEAD to signify that the flag is internal.

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Kent Overstreet <kmo@daterainc.com>
This commit is contained in:
Tejun Heo 2014-09-24 13:31:48 -04:00
parent 6251f9976a
commit eecc16ba9a
2 changed files with 52 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
include/linux/percpu-refcount.h
View file

@ -13,7 +13,7 @@
* *
* The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
* than an atomic_t - this is because of the way shutdown works, see * than an atomic_t - this is because of the way shutdown works, see
* percpu_ref_kill()/PCPU_COUNT_BIAS. * percpu_ref_kill()/PERCPU_COUNT_BIAS.
* *
* Before you call percpu_ref_kill(), percpu_ref_put() does not check for the * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
* refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
@ -60,7 +60,7 @@ struct percpu_ref {
* The low bit of the pointer indicates whether the ref is in percpu * The low bit of the pointer indicates whether the ref is in percpu
* mode; if set, then get/put will manipulate the atomic_t. * mode; if set, then get/put will manipulate the atomic_t.
*/ */
unsigned long pcpu_count_ptr; unsigned long percpu_count_ptr;
percpu_ref_func_t *release; percpu_ref_func_t *release;
percpu_ref_func_t *confirm_kill; percpu_ref_func_t *confirm_kill;
struct rcu_head rcu; struct rcu_head rcu;
@ -88,26 +88,26 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
return percpu_ref_kill_and_confirm(ref, NULL); return percpu_ref_kill_and_confirm(ref, NULL);
} }
#define PCPU_REF_DEAD 1 #define __PERCPU_REF_DEAD 1
/* /*
* Internal helper. Don't use outside percpu-refcount proper. The * Internal helper. Don't use outside percpu-refcount proper. The
* function doesn't return the pointer and let the caller test it for NULL * function doesn't return the pointer and let the caller test it for NULL
* because doing so forces the compiler to generate two conditional * because doing so forces the compiler to generate two conditional
* branches as it can't assume that @ref->pcpu_count is not NULL. * branches as it can't assume that @ref->percpu_count is not NULL.
*/ */
static inline bool __pcpu_ref_alive(struct percpu_ref *ref, static inline bool __percpu_ref_alive(struct percpu_ref *ref,
unsigned long __percpu **pcpu_countp) unsigned long __percpu **percpu_countp)
{ {
unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); unsigned long percpu_ptr = ACCESS_ONCE(ref->percpu_count_ptr);
/* paired with smp_store_release() in percpu_ref_reinit() */ /* paired with smp_store_release() in percpu_ref_reinit() */
smp_read_barrier_depends(); smp_read_barrier_depends();
if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) if (unlikely(percpu_ptr & __PERCPU_REF_DEAD))
return false; return false;
*pcpu_countp = (unsigned long __percpu *)pcpu_ptr; *percpu_countp = (unsigned long __percpu *)percpu_ptr;
return true; return true;
} }
@ -121,12 +121,12 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
*/ */
static inline void percpu_ref_get(struct percpu_ref *ref) static inline void percpu_ref_get(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count; unsigned long __percpu *percpu_count;
rcu_read_lock_sched(); rcu_read_lock_sched();
if (__pcpu_ref_alive(ref, &pcpu_count)) if (__percpu_ref_alive(ref, &percpu_count))
this_cpu_inc(*pcpu_count); this_cpu_inc(*percpu_count);
else else
atomic_long_inc(&ref->count); atomic_long_inc(&ref->count);
@ -144,13 +144,13 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
*/ */
static inline bool percpu_ref_tryget(struct percpu_ref *ref) static inline bool percpu_ref_tryget(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count; unsigned long __percpu *percpu_count;
int ret; int ret;
rcu_read_lock_sched(); rcu_read_lock_sched();
if (__pcpu_ref_alive(ref, &pcpu_count)) { if (__percpu_ref_alive(ref, &percpu_count)) {
this_cpu_inc(*pcpu_count); this_cpu_inc(*percpu_count);
ret = true; ret = true;
} else { } else {
ret = atomic_long_inc_not_zero(&ref->count); ret = atomic_long_inc_not_zero(&ref->count);
@ -178,13 +178,13 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
*/ */
static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count; unsigned long __percpu *percpu_count;
int ret = false; int ret = false;
rcu_read_lock_sched(); rcu_read_lock_sched();
if (__pcpu_ref_alive(ref, &pcpu_count)) { if (__percpu_ref_alive(ref, &percpu_count)) {
this_cpu_inc(*pcpu_count); this_cpu_inc(*percpu_count);
ret = true; ret = true;
} }
@ -204,12 +204,12 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
*/ */
static inline void percpu_ref_put(struct percpu_ref *ref) static inline void percpu_ref_put(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count; unsigned long __percpu *percpu_count;
rcu_read_lock_sched(); rcu_read_lock_sched();
if (__pcpu_ref_alive(ref, &pcpu_count)) if (__percpu_ref_alive(ref, &percpu_count))
this_cpu_dec(*pcpu_count); this_cpu_dec(*percpu_count);
else if (unlikely(atomic_long_dec_and_test(&ref->count))) else if (unlikely(atomic_long_dec_and_test(&ref->count)))
ref->release(ref); ref->release(ref);
@ -226,9 +226,9 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
*/ */
static inline bool percpu_ref_is_zero(struct percpu_ref *ref) static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count; unsigned long __percpu *percpu_count;
if (__pcpu_ref_alive(ref, &pcpu_count)) if (__percpu_ref_alive(ref, &percpu_count))
return false; return false;
return !atomic_long_read(&ref->count); return !atomic_long_read(&ref->count);
} }

56
lib/percpu-refcount.c
View file

@ -11,8 +11,8 @@
* percpu counters will all sum to the correct value * percpu counters will all sum to the correct value
* *
* (More precisely: because moduler arithmatic is commutative the sum of all the * (More precisely: because moduler arithmatic is commutative the sum of all the
* pcpu_count vars will be equal to what it would have been if all the gets and * percpu_count vars will be equal to what it would have been if all the gets
* puts were done to a single integer, even if some of the percpu integers * and puts were done to a single integer, even if some of the percpu integers
* overflow or underflow). * overflow or underflow).
* *
* The real trick to implementing percpu refcounts is shutdown. We can't detect * The real trick to implementing percpu refcounts is shutdown. We can't detect
@ -29,11 +29,12 @@
* atomic_long_t can't hit 0 before we've added up all the percpu refs. * atomic_long_t can't hit 0 before we've added up all the percpu refs.
*/ */
#define PCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1)) #define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref) static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref)
{ {
return (unsigned long __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD); return (unsigned long __percpu *)
(ref->percpu_count_ptr & ~__PERCPU_REF_DEAD);
} }
/** /**
@ -51,10 +52,11 @@ static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref)
int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
gfp_t gfp) gfp_t gfp)
{ {
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS); atomic_long_set(&ref->count, 1 + PERCPU_COUNT_BIAS);
ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned long, gfp); ref->percpu_count_ptr =
if (!ref->pcpu_count_ptr) (unsigned long)alloc_percpu_gfp(unsigned long, gfp);
if (!ref->percpu_count_ptr)
return -ENOMEM; return -ENOMEM;
ref->release = release; ref->release = release;
@ -74,11 +76,11 @@ EXPORT_SYMBOL_GPL(percpu_ref_init);
*/ */
void percpu_ref_exit(struct percpu_ref *ref) void percpu_ref_exit(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
if (pcpu_count) { if (percpu_count) {
free_percpu(pcpu_count); free_percpu(percpu_count);
ref->pcpu_count_ptr = PCPU_REF_DEAD; ref->percpu_count_ptr = __PERCPU_REF_DEAD;
} }
} }
EXPORT_SYMBOL_GPL(percpu_ref_exit); EXPORT_SYMBOL_GPL(percpu_ref_exit);
@ -86,14 +88,14 @@ EXPORT_SYMBOL_GPL(percpu_ref_exit);
static void percpu_ref_kill_rcu(struct rcu_head *rcu) static void percpu_ref_kill_rcu(struct rcu_head *rcu)
{ {
struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
unsigned long count = 0; unsigned long count = 0;
int cpu; int cpu;
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
count += *per_cpu_ptr(pcpu_count, cpu); count += *per_cpu_ptr(percpu_count, cpu);
pr_debug("global %ld pcpu %ld", pr_debug("global %ld percpu %ld",
atomic_long_read(&ref->count), (long)count); atomic_long_read(&ref->count), (long)count);
/* /*
@ -108,7 +110,7 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
* reaching 0 before we add the percpu counts. But doing it at the same * reaching 0 before we add the percpu counts. But doing it at the same
* time is equivalent and saves us atomic operations: * time is equivalent and saves us atomic operations:
*/ */
atomic_long_add((long)count - PCPU_COUNT_BIAS, &ref->count); atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count);
WARN_ONCE(atomic_long_read(&ref->count) <= 0, WARN_ONCE(atomic_long_read(&ref->count) <= 0,
"percpu ref (%pf) <= 0 (%ld) after killed", "percpu ref (%pf) <= 0 (%ld) after killed",
@ -143,10 +145,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
void percpu_ref_kill_and_confirm(struct percpu_ref *ref, void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
percpu_ref_func_t *confirm_kill) percpu_ref_func_t *confirm_kill)
{ {
WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD, WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD,
"%s called more than once on %pf!", __func__, ref->release); "%s called more than once on %pf!", __func__, ref->release);
ref->pcpu_count_ptr |= PCPU_REF_DEAD; ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
ref->confirm_kill = confirm_kill; ref->confirm_kill = confirm_kill;
call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu); call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);
@ -166,24 +168,24 @@ EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
*/ */
void percpu_ref_reinit(struct percpu_ref *ref) void percpu_ref_reinit(struct percpu_ref *ref)
{ {
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
int cpu; int cpu;
BUG_ON(!pcpu_count); BUG_ON(!percpu_count);
WARN_ON_ONCE(!percpu_ref_is_zero(ref)); WARN_ON_ONCE(!percpu_ref_is_zero(ref));
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS); atomic_long_set(&ref->count, 1 + PERCPU_COUNT_BIAS);
/* /*
* Restore per-cpu operation. smp_store_release() is paired with * Restore per-cpu operation. smp_store_release() is paired with
* smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees * smp_read_barrier_depends() in __percpu_ref_alive() and
* that the zeroing is visible to all percpu accesses which can see * guarantees that the zeroing is visible to all percpu accesses
* the following PCPU_REF_DEAD clearing. * which can see the following __PERCPU_REF_DEAD clearing.
*/ */
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
*per_cpu_ptr(pcpu_count, cpu) = 0; *per_cpu_ptr(percpu_count, cpu) = 0;
smp_store_release(&ref->pcpu_count_ptr, smp_store_release(&ref->percpu_count_ptr,
ref->pcpu_count_ptr & ~PCPU_REF_DEAD); ref->percpu_count_ptr & ~__PERCPU_REF_DEAD);
} }
EXPORT_SYMBOL_GPL(percpu_ref_reinit); EXPORT_SYMBOL_GPL(percpu_ref_reinit);