mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-14 06:35:12 +00:00
63355b9884
Commit596ff4a09b("cpumask: re-introduce constant-sized cpumask optimizations") changed cpumask_setall() to use "bitmap_set()" instead of "bitmap_fill()", because bitmap_fill() would explicitly set all the bits of a constant sized small bitmap, and that's exactly what we don't want: we want to only set bits up to 'nr_cpu_ids', which is what "bitmap_set()" does. However, Yury correctly points out that while "bitmap_set()" does indeed only set bits up to the required bitmap size, it doesn't _clear_ bits above that size, so the upper bits would still not have well-defined values. Now, none of this should really matter, since any bits set past 'nr_cpu_ids' should always be ignored in the first place. Yes, the bit scanning functions might return them as a result, but since users should always consider the ">= nr_cpu_ids" condition to mean "no more bits", that shouldn't have any actual effect (see previous commit8ca09d5fa3"cpumask: fix incorrect cpumask scanning result checks"). But let's just do it right, the way the code was _intended_ to work. We have had enough lazy code that works but bites us in the *rse later (again, see previous commit) that there's no reason to not just do this properly. It turns out that "bitmap_fill()" gets this all right for the complex case, and really only fails for the inlined optimized case that just fills the whole word. And while we could just fix bitmap_fill() to use the proper last word mask, there's two issues with that: - the cpumask case wants to do the _optimization_ based on "NR_CPUS is a small constant", but then wants to do the actual bit _fill_ based on "nr_cpu_ids" that isn't necessarily that same constant - we have lots of non-cpumask users of bitmap_fill(), and while they hopefully don't care, and probably would want the proper semantics anyway ("only set bits up to the limit"), I do not want the cpumask changes to impact other parts So this ends up just doing the single-word optimization by hand in the cpumask code. If our cpumask is fundamentally limited to a single word, just do the proper "fill in that word" exactly. And if it's the more complex multi-word case, then the generic bitmap_fill() will DTRT. This is all an example of how our bitmap function optimizations really are somewhat broken. They conflate the "this is size of the bitmap" optimizations with the actual bit(s) we want to set. In many cases we really want to have the two be separate things: sometimes we base our optimizations on the size of the whole bitmap ("I know this whole bitmap fits in a single word, so I'll just use single-word accesses"), and sometimes we base them on the bit we are looking at ("this is just acting on bits that are in the first word, so I'll use single-word accesses"). Notice how the end result of the two optimizations are the same, but the way we get to them are quite different. And all our cpumask optimization games are really about that fundamental distinction, and we'd often really want to pass in both the "this is the bit I'm working on" (which _can_ be a small constant but might be variable), and "I know it's in this range even if it's variable" (based on CONFIG_NR_CPUS). So this cpumask_setall() implementation just makes that explicit. It checks the "I statically know the size is small" using the known static size of the cpumask (which is what that 'small_cpumask_bits' is all about), but then sets the actual bits using the exact number of cpus we have (ie 'nr_cpumask_bits') Of course, in a perfect world, the compiler would have done all the range analysis (possibly with help from us just telling it that "this value is always in this range"), and would do all of this for us. But that is not the world we live in. While we dream of that perfect world, this does that manual logic to make it all work out. And this was a very long explanation for a small code change that shouldn't even matter. Reported-by: Yury Norov <yury.norov@gmail.com> Link: https://lore.kernel.org/lkml/ZAV9nGG9e1%2FrV+L%2F@yury-laptop/ Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1237 lines
35 KiB
C
1237 lines
35 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __LINUX_CPUMASK_H
|
|
#define __LINUX_CPUMASK_H
|
|
|
|
/*
|
|
* Cpumasks provide a bitmap suitable for representing the
|
|
* set of CPU's in a system, one bit position per CPU number. In general,
|
|
* only nr_cpu_ids (<= NR_CPUS) bits are valid.
|
|
*/
|
|
#include <linux/kernel.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/bitmap.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/gfp_types.h>
|
|
#include <linux/numa.h>
|
|
|
|
/* Don't assign or return these: may not be this big! */
|
|
typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
|
|
|
|
/**
|
|
* cpumask_bits - get the bits in a cpumask
|
|
* @maskp: the struct cpumask *
|
|
*
|
|
* You should only assume nr_cpu_ids bits of this mask are valid. This is
|
|
* a macro so it's const-correct.
|
|
*/
|
|
#define cpumask_bits(maskp) ((maskp)->bits)
|
|
|
|
/**
|
|
* cpumask_pr_args - printf args to output a cpumask
|
|
* @maskp: cpumask to be printed
|
|
*
|
|
* Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
|
|
*/
|
|
#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
|
|
|
|
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
|
|
#define nr_cpu_ids ((unsigned int)NR_CPUS)
|
|
#else
|
|
extern unsigned int nr_cpu_ids;
|
|
#endif
|
|
|
|
static inline void set_nr_cpu_ids(unsigned int nr)
|
|
{
|
|
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
|
|
WARN_ON(nr != nr_cpu_ids);
|
|
#else
|
|
nr_cpu_ids = nr;
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* We have several different "preferred sizes" for the cpumask
|
|
* operations, depending on operation.
|
|
*
|
|
* For example, the bitmap scanning and operating operations have
|
|
* optimized routines that work for the single-word case, but only when
|
|
* the size is constant. So if NR_CPUS fits in one single word, we are
|
|
* better off using that small constant, in order to trigger the
|
|
* optimized bit finding. That is 'small_cpumask_size'.
|
|
*
|
|
* The clearing and copying operations will similarly perform better
|
|
* with a constant size, but we limit that size arbitrarily to four
|
|
* words. We call this 'large_cpumask_size'.
|
|
*
|
|
* Finally, some operations just want the exact limit, either because
|
|
* they set bits or just don't have any faster fixed-sized versions. We
|
|
* call this just 'nr_cpumask_bits'.
|
|
*
|
|
* Note that these optional constants are always guaranteed to be at
|
|
* least as big as 'nr_cpu_ids' itself is, and all our cpumask
|
|
* allocations are at least that size (see cpumask_size()). The
|
|
* optimization comes from being able to potentially use a compile-time
|
|
* constant instead of a run-time generated exact number of CPUs.
|
|
*/
|
|
#if NR_CPUS <= BITS_PER_LONG
|
|
#define small_cpumask_bits ((unsigned int)NR_CPUS)
|
|
#define large_cpumask_bits ((unsigned int)NR_CPUS)
|
|
#elif NR_CPUS <= 4*BITS_PER_LONG
|
|
#define small_cpumask_bits nr_cpu_ids
|
|
#define large_cpumask_bits ((unsigned int)NR_CPUS)
|
|
#else
|
|
#define small_cpumask_bits nr_cpu_ids
|
|
#define large_cpumask_bits nr_cpu_ids
|
|
#endif
|
|
#define nr_cpumask_bits nr_cpu_ids
|
|
|
|
/*
|
|
* The following particular system cpumasks and operations manage
|
|
* possible, present, active and online cpus.
|
|
*
|
|
* cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
|
|
* cpu_present_mask - has bit 'cpu' set iff cpu is populated
|
|
* cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
|
|
* cpu_active_mask - has bit 'cpu' set iff cpu available to migration
|
|
*
|
|
* If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
|
|
*
|
|
* The cpu_possible_mask is fixed at boot time, as the set of CPU id's
|
|
* that it is possible might ever be plugged in at anytime during the
|
|
* life of that system boot. The cpu_present_mask is dynamic(*),
|
|
* representing which CPUs are currently plugged in. And
|
|
* cpu_online_mask is the dynamic subset of cpu_present_mask,
|
|
* indicating those CPUs available for scheduling.
|
|
*
|
|
* If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
|
|
* depending on what ACPI reports as currently plugged in, otherwise
|
|
* cpu_present_mask is just a copy of cpu_possible_mask.
|
|
*
|
|
* (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
|
|
* hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
|
|
*
|
|
* Subtleties:
|
|
* 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
|
|
* assumption that their single CPU is online. The UP
|
|
* cpu_{online,possible,present}_masks are placebos. Changing them
|
|
* will have no useful affect on the following num_*_cpus()
|
|
* and cpu_*() macros in the UP case. This ugliness is a UP
|
|
* optimization - don't waste any instructions or memory references
|
|
* asking if you're online or how many CPUs there are if there is
|
|
* only one CPU.
|
|
*/
|
|
|
|
extern struct cpumask __cpu_possible_mask;
|
|
extern struct cpumask __cpu_online_mask;
|
|
extern struct cpumask __cpu_present_mask;
|
|
extern struct cpumask __cpu_active_mask;
|
|
extern struct cpumask __cpu_dying_mask;
|
|
#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
|
|
#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
|
|
#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
|
|
#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
|
|
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
|
|
|
|
extern atomic_t __num_online_cpus;
|
|
|
|
extern cpumask_t cpus_booted_once_mask;
|
|
|
|
static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
|
|
{
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
WARN_ON_ONCE(cpu >= bits);
|
|
#endif /* CONFIG_DEBUG_PER_CPU_MAPS */
|
|
}
|
|
|
|
/* verify cpu argument to cpumask_* operators */
|
|
static __always_inline unsigned int cpumask_check(unsigned int cpu)
|
|
{
|
|
cpu_max_bits_warn(cpu, nr_cpumask_bits);
|
|
return cpu;
|
|
}
|
|
|
|
/**
|
|
* cpumask_first - get the first cpu in a cpumask
|
|
* @srcp: the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
*/
|
|
static inline unsigned int cpumask_first(const struct cpumask *srcp)
|
|
{
|
|
return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_first_zero - get the first unset cpu in a cpumask
|
|
* @srcp: the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if all cpus are set.
|
|
*/
|
|
static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
|
|
{
|
|
return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_first_and - return the first cpu from *srcp1 & *srcp2
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*
|
|
* Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
|
|
{
|
|
return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_last - get the last CPU in a cpumask
|
|
* @srcp: - the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpumask_bits if no CPUs set.
|
|
*/
|
|
static inline unsigned int cpumask_last(const struct cpumask *srcp)
|
|
{
|
|
return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_next - get the next cpu in a cpumask
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
* @srcp: the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no further cpus set.
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
|
{
|
|
/* -1 is a legal arg here. */
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
return find_next_bit(cpumask_bits(srcp), small_cpumask_bits, n + 1);
|
|
}
|
|
|
|
/**
|
|
* cpumask_next_zero - get the next unset cpu in a cpumask
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
* @srcp: the cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no further cpus unset.
|
|
*/
|
|
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
|
{
|
|
/* -1 is a legal arg here. */
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
return find_next_zero_bit(cpumask_bits(srcp), small_cpumask_bits, n+1);
|
|
}
|
|
|
|
#if NR_CPUS == 1
|
|
/* Uniprocessor: there is only one valid CPU */
|
|
static inline unsigned int cpumask_local_spread(unsigned int i, int node)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return cpumask_first_and(src1p, src2p);
|
|
}
|
|
|
|
static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
|
|
{
|
|
return cpumask_first(srcp);
|
|
}
|
|
#else
|
|
unsigned int cpumask_local_spread(unsigned int i, int node);
|
|
unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
|
|
const struct cpumask *src2p);
|
|
unsigned int cpumask_any_distribute(const struct cpumask *srcp);
|
|
#endif /* NR_CPUS */
|
|
|
|
/**
|
|
* cpumask_next_and - get the next cpu in *src1p & *src2p
|
|
* @n: the cpu prior to the place to search (ie. return will be > @n)
|
|
* @src1p: the first cpumask pointer
|
|
* @src2p: the second cpumask pointer
|
|
*
|
|
* Returns >= nr_cpu_ids if no further cpus set in both.
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
/* -1 is a legal arg here. */
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
small_cpumask_bits, n + 1);
|
|
}
|
|
|
|
/**
|
|
* for_each_cpu - iterate over every cpu in a mask
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
* @mask: the cpumask pointer
|
|
*
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
*/
|
|
#define for_each_cpu(cpu, mask) \
|
|
for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
|
|
|
|
#if NR_CPUS == 1
|
|
static inline
|
|
unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
|
|
{
|
|
cpumask_check(start);
|
|
if (n != -1)
|
|
cpumask_check(n);
|
|
|
|
/*
|
|
* Return the first available CPU when wrapping, or when starting before cpu0,
|
|
* since there is only one valid option.
|
|
*/
|
|
if (wrap && n >= 0)
|
|
return nr_cpumask_bits;
|
|
|
|
return cpumask_first(mask);
|
|
}
|
|
#else
|
|
unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
|
|
#endif
|
|
|
|
/**
|
|
* for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
* @mask: the cpumask pointer
|
|
* @start: the start location
|
|
*
|
|
* The implementation does not assume any bit in @mask is set (including @start).
|
|
*
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
*/
|
|
#define for_each_cpu_wrap(cpu, mask, start) \
|
|
for_each_set_bit_wrap(cpu, cpumask_bits(mask), small_cpumask_bits, start)
|
|
|
|
/**
|
|
* for_each_cpu_and - iterate over every cpu in both masks
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
* @mask1: the first cpumask pointer
|
|
* @mask2: the second cpumask pointer
|
|
*
|
|
* This saves a temporary CPU mask in many places. It is equivalent to:
|
|
* struct cpumask tmp;
|
|
* cpumask_and(&tmp, &mask1, &mask2);
|
|
* for_each_cpu(cpu, &tmp)
|
|
* ...
|
|
*
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
*/
|
|
#define for_each_cpu_and(cpu, mask1, mask2) \
|
|
for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
|
|
|
|
/**
|
|
* for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
|
|
* those present in another.
|
|
* @cpu: the (optionally unsigned) integer iterator
|
|
* @mask1: the first cpumask pointer
|
|
* @mask2: the second cpumask pointer
|
|
*
|
|
* This saves a temporary CPU mask in many places. It is equivalent to:
|
|
* struct cpumask tmp;
|
|
* cpumask_andnot(&tmp, &mask1, &mask2);
|
|
* for_each_cpu(cpu, &tmp)
|
|
* ...
|
|
*
|
|
* After the loop, cpu is >= nr_cpu_ids.
|
|
*/
|
|
#define for_each_cpu_andnot(cpu, mask1, mask2) \
|
|
for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
|
|
|
|
/**
|
|
* cpumask_any_but - return a "random" in a cpumask, but not this one.
|
|
* @mask: the cpumask to search
|
|
* @cpu: the cpu to ignore.
|
|
*
|
|
* Often used to find any cpu but smp_processor_id() in a mask.
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
|
|
{
|
|
unsigned int i;
|
|
|
|
cpumask_check(cpu);
|
|
for_each_cpu(i, mask)
|
|
if (i != cpu)
|
|
break;
|
|
return i;
|
|
}
|
|
|
|
/**
|
|
* cpumask_nth - get the first cpu in a cpumask
|
|
* @srcp: the cpumask pointer
|
|
* @cpu: the N'th cpu to find, starting from 0
|
|
*
|
|
* Returns >= nr_cpu_ids if such cpu doesn't exist.
|
|
*/
|
|
static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
|
|
{
|
|
return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
|
|
}
|
|
|
|
/**
|
|
* cpumask_nth_and - get the first cpu in 2 cpumasks
|
|
* @srcp1: the cpumask pointer
|
|
* @srcp2: the cpumask pointer
|
|
* @cpu: the N'th cpu to find, starting from 0
|
|
*
|
|
* Returns >= nr_cpu_ids if such cpu doesn't exist.
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
|
|
const struct cpumask *srcp2)
|
|
{
|
|
return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
|
|
small_cpumask_bits, cpumask_check(cpu));
|
|
}
|
|
|
|
/**
|
|
* cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
|
|
* @srcp1: the cpumask pointer
|
|
* @srcp2: the cpumask pointer
|
|
* @cpu: the N'th cpu to find, starting from 0
|
|
*
|
|
* Returns >= nr_cpu_ids if such cpu doesn't exist.
|
|
*/
|
|
static inline
|
|
unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
|
|
const struct cpumask *srcp2)
|
|
{
|
|
return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
|
|
small_cpumask_bits, cpumask_check(cpu));
|
|
}
|
|
|
|
/**
|
|
* cpumask_nth_and_andnot - get the Nth cpu set in 1st and 2nd cpumask, and clear in 3rd.
|
|
* @srcp1: the cpumask pointer
|
|
* @srcp2: the cpumask pointer
|
|
* @srcp3: the cpumask pointer
|
|
* @cpu: the N'th cpu to find, starting from 0
|
|
*
|
|
* Returns >= nr_cpu_ids if such cpu doesn't exist.
|
|
*/
|
|
static __always_inline
|
|
unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp1,
|
|
const struct cpumask *srcp2,
|
|
const struct cpumask *srcp3)
|
|
{
|
|
return find_nth_and_andnot_bit(cpumask_bits(srcp1),
|
|
cpumask_bits(srcp2),
|
|
cpumask_bits(srcp3),
|
|
small_cpumask_bits, cpumask_check(cpu));
|
|
}
|
|
|
|
#define CPU_BITS_NONE \
|
|
{ \
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
|
|
}
|
|
|
|
#define CPU_BITS_CPU0 \
|
|
{ \
|
|
[0] = 1UL \
|
|
}
|
|
|
|
/**
|
|
* cpumask_set_cpu - set a cpu in a cpumask
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
* @dstp: the cpumask pointer
|
|
*/
|
|
static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
|
{
|
|
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
}
|
|
|
|
static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
|
{
|
|
__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
}
|
|
|
|
|
|
/**
|
|
* cpumask_clear_cpu - clear a cpu in a cpumask
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
* @dstp: the cpumask pointer
|
|
*/
|
|
static __always_inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
|
|
{
|
|
clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
}
|
|
|
|
static __always_inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
|
|
{
|
|
__clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
|
}
|
|
|
|
/**
|
|
* cpumask_test_cpu - test for a cpu in a cpumask
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
* @cpumask: the cpumask pointer
|
|
*
|
|
* Returns true if @cpu is set in @cpumask, else returns false
|
|
*/
|
|
static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
|
|
{
|
|
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
|
|
}
|
|
|
|
/**
|
|
* cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
* @cpumask: the cpumask pointer
|
|
*
|
|
* Returns true if @cpu is set in old bitmap of @cpumask, else returns false
|
|
*
|
|
* test_and_set_bit wrapper for cpumasks.
|
|
*/
|
|
static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
|
|
{
|
|
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
|
}
|
|
|
|
/**
|
|
* cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
|
|
* @cpu: cpu number (< nr_cpu_ids)
|
|
* @cpumask: the cpumask pointer
|
|
*
|
|
* Returns true if @cpu is set in old bitmap of @cpumask, else returns false
|
|
*
|
|
* test_and_clear_bit wrapper for cpumasks.
|
|
*/
|
|
static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
|
|
{
|
|
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
|
}
|
|
|
|
/**
|
|
* cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
|
|
* @dstp: the cpumask pointer
|
|
*/
|
|
static inline void cpumask_setall(struct cpumask *dstp)
|
|
{
|
|
if (small_const_nbits(small_cpumask_bits)) {
|
|
cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
|
|
return;
|
|
}
|
|
bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
|
|
* @dstp: the cpumask pointer
|
|
*/
|
|
static inline void cpumask_clear(struct cpumask *dstp)
|
|
{
|
|
bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_and - *dstp = *src1p & *src2p
|
|
* @dstp: the cpumask result
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*
|
|
* If *@dstp is empty, returns false, else returns true
|
|
*/
|
|
static inline bool cpumask_and(struct cpumask *dstp,
|
|
const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
cpumask_bits(src2p), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_or - *dstp = *src1p | *src2p
|
|
* @dstp: the cpumask result
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*/
|
|
static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
cpumask_bits(src2p), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_xor - *dstp = *src1p ^ *src2p
|
|
* @dstp: the cpumask result
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*/
|
|
static inline void cpumask_xor(struct cpumask *dstp,
|
|
const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
cpumask_bits(src2p), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_andnot - *dstp = *src1p & ~*src2p
|
|
* @dstp: the cpumask result
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*
|
|
* If *@dstp is empty, returns false, else returns true
|
|
*/
|
|
static inline bool cpumask_andnot(struct cpumask *dstp,
|
|
const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
|
|
cpumask_bits(src2p), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_equal - *src1p == *src2p
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*/
|
|
static inline bool cpumask_equal(const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_or_equal - *src1p | *src2p == *src3p
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
* @src3p: the third input
|
|
*/
|
|
static inline bool cpumask_or_equal(const struct cpumask *src1p,
|
|
const struct cpumask *src2p,
|
|
const struct cpumask *src3p)
|
|
{
|
|
return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
cpumask_bits(src3p), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_intersects - (*src1p & *src2p) != 0
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*/
|
|
static inline bool cpumask_intersects(const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_subset - (*src1p & ~*src2p) == 0
|
|
* @src1p: the first input
|
|
* @src2p: the second input
|
|
*
|
|
* Returns true if *@src1p is a subset of *@src2p, else returns false
|
|
*/
|
|
static inline bool cpumask_subset(const struct cpumask *src1p,
|
|
const struct cpumask *src2p)
|
|
{
|
|
return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
|
|
small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_empty - *srcp == 0
|
|
* @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
|
|
*/
|
|
static inline bool cpumask_empty(const struct cpumask *srcp)
|
|
{
|
|
return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_full - *srcp == 0xFFFFFFFF...
|
|
* @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
|
|
*/
|
|
static inline bool cpumask_full(const struct cpumask *srcp)
|
|
{
|
|
return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_weight - Count of bits in *srcp
|
|
* @srcp: the cpumask to count bits (< nr_cpu_ids) in.
|
|
*/
|
|
static inline unsigned int cpumask_weight(const struct cpumask *srcp)
|
|
{
|
|
return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
|
|
* @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
|
|
* @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
|
|
*/
|
|
static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
|
|
const struct cpumask *srcp2)
|
|
{
|
|
return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_shift_right - *dstp = *srcp >> n
|
|
* @dstp: the cpumask result
|
|
* @srcp: the input to shift
|
|
* @n: the number of bits to shift by
|
|
*/
|
|
static inline void cpumask_shift_right(struct cpumask *dstp,
|
|
const struct cpumask *srcp, int n)
|
|
{
|
|
bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
|
small_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_shift_left - *dstp = *srcp << n
|
|
* @dstp: the cpumask result
|
|
* @srcp: the input to shift
|
|
* @n: the number of bits to shift by
|
|
*/
|
|
static inline void cpumask_shift_left(struct cpumask *dstp,
|
|
const struct cpumask *srcp, int n)
|
|
{
|
|
bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
|
nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_copy - *dstp = *srcp
|
|
* @dstp: the result
|
|
* @srcp: the input cpumask
|
|
*/
|
|
static inline void cpumask_copy(struct cpumask *dstp,
|
|
const struct cpumask *srcp)
|
|
{
|
|
bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_any - pick a "random" cpu from *srcp
|
|
* @srcp: the input cpumask
|
|
*
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
*/
|
|
#define cpumask_any(srcp) cpumask_first(srcp)
|
|
|
|
/**
|
|
* cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
|
|
* @mask1: the first input cpumask
|
|
* @mask2: the second input cpumask
|
|
*
|
|
* Returns >= nr_cpu_ids if no cpus set.
|
|
*/
|
|
#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
|
|
|
|
/**
|
|
* cpumask_of - the cpumask containing just a given cpu
|
|
* @cpu: the cpu (<= nr_cpu_ids)
|
|
*/
|
|
#define cpumask_of(cpu) (get_cpu_mask(cpu))
|
|
|
|
/**
|
|
* cpumask_parse_user - extract a cpumask from a user string
|
|
* @buf: the buffer to extract from
|
|
* @len: the length of the buffer
|
|
* @dstp: the cpumask to set.
|
|
*
|
|
* Returns -errno, or 0 for success.
|
|
*/
|
|
static inline int cpumask_parse_user(const char __user *buf, int len,
|
|
struct cpumask *dstp)
|
|
{
|
|
return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_parselist_user - extract a cpumask from a user string
|
|
* @buf: the buffer to extract from
|
|
* @len: the length of the buffer
|
|
* @dstp: the cpumask to set.
|
|
*
|
|
* Returns -errno, or 0 for success.
|
|
*/
|
|
static inline int cpumask_parselist_user(const char __user *buf, int len,
|
|
struct cpumask *dstp)
|
|
{
|
|
return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
|
|
nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_parse - extract a cpumask from a string
|
|
* @buf: the buffer to extract from
|
|
* @dstp: the cpumask to set.
|
|
*
|
|
* Returns -errno, or 0 for success.
|
|
*/
|
|
static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
|
|
{
|
|
return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpulist_parse - extract a cpumask from a user string of ranges
|
|
* @buf: the buffer to extract from
|
|
* @dstp: the cpumask to set.
|
|
*
|
|
* Returns -errno, or 0 for success.
|
|
*/
|
|
static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
|
|
{
|
|
return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
|
|
}
|
|
|
|
/**
|
|
* cpumask_size - size to allocate for a 'struct cpumask' in bytes
|
|
*/
|
|
static inline unsigned int cpumask_size(void)
|
|
{
|
|
return BITS_TO_LONGS(large_cpumask_bits) * sizeof(long);
|
|
}
|
|
|
|
/*
|
|
* cpumask_var_t: struct cpumask for stack usage.
|
|
*
|
|
* Oh, the wicked games we play! In order to make kernel coding a
|
|
* little more difficult, we typedef cpumask_var_t to an array or a
|
|
* pointer: doing &mask on an array is a noop, so it still works.
|
|
*
|
|
* ie.
|
|
* cpumask_var_t tmpmask;
|
|
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
|
|
* return -ENOMEM;
|
|
*
|
|
* ... use 'tmpmask' like a normal struct cpumask * ...
|
|
*
|
|
* free_cpumask_var(tmpmask);
|
|
*
|
|
*
|
|
* However, one notable exception is there. alloc_cpumask_var() allocates
|
|
* only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
|
|
* NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
|
|
*
|
|
* cpumask_var_t tmpmask;
|
|
* if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
|
|
* return -ENOMEM;
|
|
*
|
|
* var = *tmpmask;
|
|
*
|
|
* This code makes NR_CPUS length memcopy and brings to a memory corruption.
|
|
* cpumask_copy() provide safe copy functionality.
|
|
*
|
|
* Note that there is another evil here: If you define a cpumask_var_t
|
|
* as a percpu variable then the way to obtain the address of the cpumask
|
|
* structure differently influences what this_cpu_* operation needs to be
|
|
* used. Please use this_cpu_cpumask_var_t in those cases. The direct use
|
|
* of this_cpu_ptr() or this_cpu_read() will lead to failures when the
|
|
* other type of cpumask_var_t implementation is configured.
|
|
*
|
|
* Please also note that __cpumask_var_read_mostly can be used to declare
|
|
* a cpumask_var_t variable itself (not its content) as read mostly.
|
|
*/
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
typedef struct cpumask *cpumask_var_t;
|
|
|
|
#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
|
|
#define __cpumask_var_read_mostly __read_mostly
|
|
|
|
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
|
|
|
|
static inline
|
|
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
|
|
{
|
|
return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
|
|
}
|
|
|
|
/**
|
|
* alloc_cpumask_var - allocate a struct cpumask
|
|
* @mask: pointer to cpumask_var_t where the cpumask is returned
|
|
* @flags: GFP_ flags
|
|
*
|
|
* Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
|
|
* a nop returning a constant 1 (in <linux/cpumask.h>).
|
|
*
|
|
* See alloc_cpumask_var_node.
|
|
*/
|
|
static inline
|
|
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
|
|
}
|
|
|
|
static inline
|
|
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
return alloc_cpumask_var(mask, flags | __GFP_ZERO);
|
|
}
|
|
|
|
void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
|
|
void free_cpumask_var(cpumask_var_t mask);
|
|
void free_bootmem_cpumask_var(cpumask_var_t mask);
|
|
|
|
static inline bool cpumask_available(cpumask_var_t mask)
|
|
{
|
|
return mask != NULL;
|
|
}
|
|
|
|
#else
|
|
typedef struct cpumask cpumask_var_t[1];
|
|
|
|
#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
|
|
#define __cpumask_var_read_mostly
|
|
|
|
static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
|
int node)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
|
{
|
|
cpumask_clear(*mask);
|
|
return true;
|
|
}
|
|
|
|
static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
|
int node)
|
|
{
|
|
cpumask_clear(*mask);
|
|
return true;
|
|
}
|
|
|
|
static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
|
|
{
|
|
}
|
|
|
|
static inline void free_cpumask_var(cpumask_var_t mask)
|
|
{
|
|
}
|
|
|
|
static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
|
|
{
|
|
}
|
|
|
|
static inline bool cpumask_available(cpumask_var_t mask)
|
|
{
|
|
return true;
|
|
}
|
|
#endif /* CONFIG_CPUMASK_OFFSTACK */
|
|
|
|
/* It's common to want to use cpu_all_mask in struct member initializers,
|
|
* so it has to refer to an address rather than a pointer. */
|
|
extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
|
|
#define cpu_all_mask to_cpumask(cpu_all_bits)
|
|
|
|
/* First bits of cpu_bit_bitmap are in fact unset. */
|
|
#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
|
|
|
|
#if NR_CPUS == 1
|
|
/* Uniprocessor: the possible/online/present masks are always "1" */
|
|
#define for_each_possible_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
|
|
#define for_each_online_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
|
|
#define for_each_present_cpu(cpu) for ((cpu) = 0; (cpu) < 1; (cpu)++)
|
|
#else
|
|
#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
|
|
#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
|
|
#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
|
|
#endif
|
|
|
|
/* Wrappers for arch boot code to manipulate normally-constant masks */
|
|
void init_cpu_present(const struct cpumask *src);
|
|
void init_cpu_possible(const struct cpumask *src);
|
|
void init_cpu_online(const struct cpumask *src);
|
|
|
|
static inline void reset_cpu_possible_mask(void)
|
|
{
|
|
bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
|
|
}
|
|
|
|
static inline void
|
|
set_cpu_possible(unsigned int cpu, bool possible)
|
|
{
|
|
if (possible)
|
|
cpumask_set_cpu(cpu, &__cpu_possible_mask);
|
|
else
|
|
cpumask_clear_cpu(cpu, &__cpu_possible_mask);
|
|
}
|
|
|
|
static inline void
|
|
set_cpu_present(unsigned int cpu, bool present)
|
|
{
|
|
if (present)
|
|
cpumask_set_cpu(cpu, &__cpu_present_mask);
|
|
else
|
|
cpumask_clear_cpu(cpu, &__cpu_present_mask);
|
|
}
|
|
|
|
void set_cpu_online(unsigned int cpu, bool online);
|
|
|
|
static inline void
|
|
set_cpu_active(unsigned int cpu, bool active)
|
|
{
|
|
if (active)
|
|
cpumask_set_cpu(cpu, &__cpu_active_mask);
|
|
else
|
|
cpumask_clear_cpu(cpu, &__cpu_active_mask);
|
|
}
|
|
|
|
static inline void
|
|
set_cpu_dying(unsigned int cpu, bool dying)
|
|
{
|
|
if (dying)
|
|
cpumask_set_cpu(cpu, &__cpu_dying_mask);
|
|
else
|
|
cpumask_clear_cpu(cpu, &__cpu_dying_mask);
|
|
}
|
|
|
|
/**
|
|
* to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
|
|
* @bitmap: the bitmap
|
|
*
|
|
* There are a few places where cpumask_var_t isn't appropriate and
|
|
* static cpumasks must be used (eg. very early boot), yet we don't
|
|
* expose the definition of 'struct cpumask'.
|
|
*
|
|
* This does the conversion, and can be used as a constant initializer.
|
|
*/
|
|
#define to_cpumask(bitmap) \
|
|
((struct cpumask *)(1 ? (bitmap) \
|
|
: (void *)sizeof(__check_is_bitmap(bitmap))))
|
|
|
|
static inline int __check_is_bitmap(const unsigned long *bitmap)
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
/*
|
|
* Special-case data structure for "single bit set only" constant CPU masks.
|
|
*
|
|
* We pre-generate all the 64 (or 32) possible bit positions, with enough
|
|
* padding to the left and the right, and return the constant pointer
|
|
* appropriately offset.
|
|
*/
|
|
extern const unsigned long
|
|
cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
|
|
|
|
static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
|
|
{
|
|
const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
|
|
p -= cpu / BITS_PER_LONG;
|
|
return to_cpumask(p);
|
|
}
|
|
|
|
#if NR_CPUS > 1
|
|
/**
|
|
* num_online_cpus() - Read the number of online CPUs
|
|
*
|
|
* Despite the fact that __num_online_cpus is of type atomic_t, this
|
|
* interface gives only a momentary snapshot and is not protected against
|
|
* concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
|
|
* region.
|
|
*/
|
|
static __always_inline unsigned int num_online_cpus(void)
|
|
{
|
|
return arch_atomic_read(&__num_online_cpus);
|
|
}
|
|
#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
|
|
#define num_present_cpus() cpumask_weight(cpu_present_mask)
|
|
#define num_active_cpus() cpumask_weight(cpu_active_mask)
|
|
|
|
static inline bool cpu_online(unsigned int cpu)
|
|
{
|
|
return cpumask_test_cpu(cpu, cpu_online_mask);
|
|
}
|
|
|
|
static inline bool cpu_possible(unsigned int cpu)
|
|
{
|
|
return cpumask_test_cpu(cpu, cpu_possible_mask);
|
|
}
|
|
|
|
static inline bool cpu_present(unsigned int cpu)
|
|
{
|
|
return cpumask_test_cpu(cpu, cpu_present_mask);
|
|
}
|
|
|
|
static inline bool cpu_active(unsigned int cpu)
|
|
{
|
|
return cpumask_test_cpu(cpu, cpu_active_mask);
|
|
}
|
|
|
|
static inline bool cpu_dying(unsigned int cpu)
|
|
{
|
|
return cpumask_test_cpu(cpu, cpu_dying_mask);
|
|
}
|
|
|
|
#else
|
|
|
|
#define num_online_cpus() 1U
|
|
#define num_possible_cpus() 1U
|
|
#define num_present_cpus() 1U
|
|
#define num_active_cpus() 1U
|
|
|
|
static inline bool cpu_online(unsigned int cpu)
|
|
{
|
|
return cpu == 0;
|
|
}
|
|
|
|
static inline bool cpu_possible(unsigned int cpu)
|
|
{
|
|
return cpu == 0;
|
|
}
|
|
|
|
static inline bool cpu_present(unsigned int cpu)
|
|
{
|
|
return cpu == 0;
|
|
}
|
|
|
|
static inline bool cpu_active(unsigned int cpu)
|
|
{
|
|
return cpu == 0;
|
|
}
|
|
|
|
static inline bool cpu_dying(unsigned int cpu)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
#endif /* NR_CPUS > 1 */
|
|
|
|
#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
|
|
|
|
#if NR_CPUS <= BITS_PER_LONG
|
|
#define CPU_BITS_ALL \
|
|
{ \
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
|
}
|
|
|
|
#else /* NR_CPUS > BITS_PER_LONG */
|
|
|
|
#define CPU_BITS_ALL \
|
|
{ \
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
|
}
|
|
#endif /* NR_CPUS > BITS_PER_LONG */
|
|
|
|
/**
|
|
* cpumap_print_to_pagebuf - copies the cpumask into the buffer either
|
|
* as comma-separated list of cpus or hex values of cpumask
|
|
* @list: indicates whether the cpumap must be list
|
|
* @mask: the cpumask to copy
|
|
* @buf: the buffer to copy into
|
|
*
|
|
* Returns the length of the (null-terminated) @buf string, zero if
|
|
* nothing is copied.
|
|
*/
|
|
static inline ssize_t
|
|
cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
|
|
{
|
|
return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
|
|
nr_cpu_ids);
|
|
}
|
|
|
|
/**
|
|
* cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as
|
|
* hex values of cpumask
|
|
*
|
|
* @buf: the buffer to copy into
|
|
* @mask: the cpumask to copy
|
|
* @off: in the string from which we are copying, we copy to @buf
|
|
* @count: the maximum number of bytes to print
|
|
*
|
|
* The function prints the cpumask into the buffer as hex values of
|
|
* cpumask; Typically used by bin_attribute to export cpumask bitmask
|
|
* ABI.
|
|
*
|
|
* Returns the length of how many bytes have been copied, excluding
|
|
* terminating '\0'.
|
|
*/
|
|
static inline ssize_t
|
|
cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
|
|
loff_t off, size_t count)
|
|
{
|
|
return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
|
|
nr_cpu_ids, off, count) - 1;
|
|
}
|
|
|
|
/**
|
|
* cpumap_print_list_to_buf - copies the cpumask into the buffer as
|
|
* comma-separated list of cpus
|
|
*
|
|
* Everything is same with the above cpumap_print_bitmask_to_buf()
|
|
* except the print format.
|
|
*/
|
|
static inline ssize_t
|
|
cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
|
|
loff_t off, size_t count)
|
|
{
|
|
return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
|
|
nr_cpu_ids, off, count) - 1;
|
|
}
|
|
|
|
#if NR_CPUS <= BITS_PER_LONG
|
|
#define CPU_MASK_ALL \
|
|
(cpumask_t) { { \
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
|
} }
|
|
#else
|
|
#define CPU_MASK_ALL \
|
|
(cpumask_t) { { \
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
|
|
[BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
|
|
} }
|
|
#endif /* NR_CPUS > BITS_PER_LONG */
|
|
|
|
#define CPU_MASK_NONE \
|
|
(cpumask_t) { { \
|
|
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
|
|
} }
|
|
|
|
#define CPU_MASK_CPU0 \
|
|
(cpumask_t) { { \
|
|
[0] = 1UL \
|
|
} }
|
|
|
|
/*
|
|
* Provide a valid theoretical max size for cpumap and cpulist sysfs files
|
|
* to avoid breaking userspace which may allocate a buffer based on the size
|
|
* reported by e.g. fstat.
|
|
*
|
|
* for cpumap NR_CPUS * 9/32 - 1 should be an exact length.
|
|
*
|
|
* For cpulist 7 is (ceil(log10(NR_CPUS)) + 1) allowing for NR_CPUS to be up
|
|
* to 2 orders of magnitude larger than 8192. And then we divide by 2 to
|
|
* cover a worst-case of every other cpu being on one of two nodes for a
|
|
* very large NR_CPUS.
|
|
*
|
|
* Use PAGE_SIZE as a minimum for smaller configurations while avoiding
|
|
* unsigned comparison to -1.
|
|
*/
|
|
#define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \
|
|
? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE)
|
|
#define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE)
|
|
|
|
#endif /* __LINUX_CPUMASK_H */
|