mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

bitmap patches for v6.1-rc1 

From Phil Auld:
 drivers/base: Fix unsigned comparison to -1 in CPUMAP_FILE_MAX_BYTES
 
 From me:
 cpumask: cleanup nr_cpu_ids vs nr_cpumask_bits mess
 
 This series cleans that mess and adds new config FORCE_NR_CPUS that
 allows to optimize cpumask subsystem if the number of CPUs is known
 at compile-time.
 
 From me:
 lib: optimize find_bit() functions
 
 Reworks find_bit() functions based on new FIND_{FIRST,NEXT}_BIT() macros.
 
 From me:
 lib/find: add find_nth_bit()
 
 Adds find_nth_bit(), which is ~70 times faster than bitcounting with
 for_each() loop:
         for_each_set_bit(bit, mask, size)
                 if (n-- == 0)
                         return bit;
 
 Also adds bitmap_weight_and() to let people replace this pattern:
 	tmp = bitmap_alloc(nbits);
 	bitmap_and(tmp, map1, map2, nbits);
 	weight = bitmap_weight(tmp, nbits);
 	bitmap_free(tmp);
 with a single bitmap_weight_and() call.
 
 From me:
 cpumask: repair cpumask_check()
 
 After switching cpumask to use nr_cpu_ids, cpumask_check() started
 generating many false-positive warnings. This series fixes it.
 
 From Valentin Schneider:
 bitmap,cpumask: Add for_each_cpu_andnot() and for_each_cpu_andnot()
 
 Extends the API with one more function and applies it in sched/core.
 -----BEGIN PGP SIGNATURE-----
 
 iQGzBAABCgAdFiEEi8GdvG6xMhdgpu/4sUSA/TofvsgFAmNBwmUACgkQsUSA/Tof
 vshPRwv+KlqnZlKtuSPgbo/Kgswworpi/7TqfnN9GWlb8AJ2uhjBKI3GFwv4TDow
 7KV6wdKdXYLr4pktcIhWy3qLrT+bDDExfarHRo3QI1A1W42EJ+ZiUaGnQGcnVMzD
 5q/K1YMJYq0oaesHEw5PVUh8mm6h9qRD8VbX1u+riW/VCWBj3bho9Dp4mffQ48Q6
 hVy/SnMGgClQwNYp+sxkqYx38xUqUGYoU5MzeziUmoS6pZQh+4lF33MULnI3EKmc
 /ehXilPPtOV/Tm0RovDWFfm3rjNapV9FXHu8Ob2z/c+1A29EgXnE3pwrBDkAx001
 TQrL9qbCANRDGPLzWQHw0dwFIaXvTdrSttCsfYYfU5hI4JbnJEe0Pqkaaohy7jqm
 r0dW/TlyOG5T+k8Kwdx9w9A+jKs8TbKKZ8HOaN8BpkXswVnpbzpQbj3TITZI4aeV
 6YR4URBQ5UkrVLEXFXbrOzwjL2zqDdyNoBdTJmGLJ+5b/n0HHzmyMVkegNIwLLM3
 GR7sMQae
 =Q/+F
 -----END PGP SIGNATURE-----

Merge tag 'bitmap-6.1-rc1' of https://github.com/norov/linux

Pull bitmap updates from Yury Norov:

 - Fix unsigned comparison to -1 in CPUMAP_FILE_MAX_BYTES (Phil Auld)

 - cleanup nr_cpu_ids vs nr_cpumask_bits mess (me)

   This series cleans that mess and adds new config FORCE_NR_CPUS that
   allows to optimize cpumask subsystem if the number of CPUs is known
   at compile-time.

 - optimize find_bit() functions (me)

   Reworks find_bit() functions based on new FIND_{FIRST,NEXT}_BIT()
   macros.

 - add find_nth_bit() (me)

   Adds find_nth_bit(), which is ~70 times faster than bitcounting with
   for_each() loop:

	for_each_set_bit(bit, mask, size)
		if (n-- == 0)
			return bit;

   Also adds bitmap_weight_and() to let people replace this pattern:

	tmp = bitmap_alloc(nbits);
	bitmap_and(tmp, map1, map2, nbits);
	weight = bitmap_weight(tmp, nbits);
	bitmap_free(tmp);

   with a single bitmap_weight_and() call.

 - repair cpumask_check() (me)

   After switching cpumask to use nr_cpu_ids, cpumask_check() started
   generating many false-positive warnings. This series fixes it.

 - Add for_each_cpu_andnot() and for_each_cpu_andnot() (Valentin
   Schneider)

   Extends the API with one more function and applies it in sched/core.

* tag 'bitmap-6.1-rc1' of https://github.com/norov/linux: (28 commits)
  sched/core: Merge cpumask_andnot()+for_each_cpu() into for_each_cpu_andnot()
  lib/test_cpumask: Add for_each_cpu_and(not) tests
  cpumask: Introduce for_each_cpu_andnot()
  lib/find_bit: Introduce find_next_andnot_bit()
  cpumask: fix checking valid cpu range
  lib/bitmap: add tests for for_each() loops
  lib/find: optimize for_each() macros
  lib/bitmap: introduce for_each_set_bit_wrap() macro
  lib/find_bit: add find_next{,_and}_bit_wrap
  cpumask: switch for_each_cpu{,_not} to use for_each_bit()
  net: fix cpu_max_bits_warn() usage in netif_attrmask_next{,_and}
  cpumask: add cpumask_nth_{,and,andnot}
  lib/bitmap: remove bitmap_ord_to_pos
  lib/bitmap: add tests for find_nth_bit()
  lib: add find_nth{,_and,_andnot}_bit()
  lib/bitmap: add bitmap_weight_and()
  lib/bitmap: don't call __bitmap_weight() in kernel code
  tools: sync find_bit() implementation
  lib/find_bit: optimize find_next_bit() functions
  lib/find_bit: create find_first_zero_bit_le()
  ...
This commit is contained in:
Linus Torvalds 2022-10-10 12:49:34 -07:00
parent cdf072acb5 585463f0d5
commit d4013bc4d4
23 changed files with 1035 additions and 369 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/loongarch/kernel/setup.c
View File

@ -336,7 +336,7 @@ static void __init prefill_possible_map(void)
for (; i < NR_CPUS; i++)
set_cpu_possible(i, false);
nr_cpu_ids = possible;
set_nr_cpu_ids(possible);
}
#endif

2
arch/mips/kernel/setup.c
View File

@ -751,7 +751,7 @@ static void __init prefill_possible_map(void)
for (; i < NR_CPUS; i++)
set_cpu_possible(i, false);
nr_cpu_ids = possible;
set_nr_cpu_ids(possible);
}
#else
static inline void prefill_possible_map(void) {}

4
arch/powerpc/kernel/head_64.S
View File

@ -393,8 +393,12 @@ generic_secondary_common_init:
#else
LOAD_REG_ADDR(r8, paca_ptrs) /* Load paca_ptrs pointe */
ld r8,0(r8) /* Get base vaddr of array */
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
LOAD_REG_IMMEDIATE(r7, NR_CPUS)
#else
LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */
lwz r7,0(r7) /* also the max paca allocated */
#endif
li r5,0 /* logical cpu id */
1:
sldi r9,r5,3 /* get paca_ptrs[] index from cpu id */

4
arch/x86/kernel/smpboot.c
View File

@ -1316,7 +1316,7 @@ static void __init smp_sanity_check(void)
nr++;
}
nr_cpu_ids = 8;
set_nr_cpu_ids(8);
}
#endif
@ -1569,7 +1569,7 @@ __init void prefill_possible_map(void)
possible = i;
}
nr_cpu_ids = possible;
set_nr_cpu_ids(possible);
pr_info("Allowing %d CPUs, %d hotplug CPUs\n",
possible, max_t(int, possible - num_processors, 0));

2
arch/x86/xen/smp_pv.c
View File

@ -179,7 +179,7 @@ static void __init _get_smp_config(unsigned int early)
* hypercall to expand the max number of VCPUs an already
* running guest has. So cap it up to X. */
if (subtract)
nr_cpu_ids = nr_cpu_ids - subtract;
set_nr_cpu_ids(nr_cpu_ids - subtract);
#endif
}

4
fs/ntfs3/bitmap.c
View File

@ -560,7 +560,7 @@ static int wnd_rescan(struct wnd_bitmap *wnd)
buf = (ulong *)bh->b_data;
used = __bitmap_weight(buf, wbits);
used = bitmap_weight(buf, wbits);
if (used < wbits) {
frb = wbits - used;
wnd->free_bits[iw] = frb;
@ -1364,7 +1364,7 @@ int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits)
buf = (ulong *)bh->b_data;
__bitmap_clear(buf, b0, blocksize * 8 - b0);
frb = wbits - __bitmap_weight(buf, wbits);
frb = wbits - bitmap_weight(buf, wbits);
wnd->total_zeroes += frb - wnd->free_bits[iw];
wnd->free_bits[iw] = frb;

13
include/linux/bitmap.h
View File

@ -51,6 +51,7 @@ struct device;
* bitmap_empty(src, nbits) Are all bits zero in *src?
* bitmap_full(src, nbits) Are all bits set in *src?
* bitmap_weight(src, nbits) Hamming Weight: number set bits
* bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap
* bitmap_set(dst, pos, nbits) Set specified bit area
* bitmap_clear(dst, pos, nbits) Clear specified bit area
* bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area
@ -164,6 +165,8 @@ bool __bitmap_intersects(const unsigned long *bitmap1,
bool __bitmap_subset(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
unsigned int __bitmap_weight_and(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_set(unsigned long *map, unsigned int start, int len);
void __bitmap_clear(unsigned long *map, unsigned int start, int len);
@ -222,7 +225,6 @@ void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int n
#else
#define bitmap_copy_le bitmap_copy
#endif
unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits);
int bitmap_print_to_pagebuf(bool list, char *buf,
const unsigned long *maskp, int nmaskbits);
@ -439,6 +441,15 @@ unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits)
return __bitmap_weight(src, nbits);
}
static __always_inline
unsigned long bitmap_weight_and(const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
if (small_const_nbits(nbits))
return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits));
return __bitmap_weight_and(src1, src2, nbits);
}
static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
unsigned int nbits)
{

19
include/linux/bitops.h
View File

@ -247,6 +247,25 @@ static inline unsigned long __ffs64(u64 word)
return __ffs((unsigned long)word);
}
/**
* fns - find N'th set bit in a word
* @word: The word to search
* @n: Bit to find
*/
static inline unsigned long fns(unsigned long word, unsigned int n)
{
unsigned int bit;
while (word) {
bit = __ffs(word);
if (n-- == 0)
return bit;
__clear_bit(bit, &word);
}
return BITS_PER_LONG;
}
/**
* assign_bit - Assign value to a bit in memory
* @nr: the bit to set

132
include/linux/cpumask.h
View File

@ -35,19 +35,23 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
*/
#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
#if NR_CPUS == 1
#define nr_cpu_ids 1U
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
#define nr_cpu_ids ((unsigned int)NR_CPUS)
#else
extern unsigned int nr_cpu_ids;
#endif
#ifdef CONFIG_CPUMASK_OFFSTACK
/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
* not all bits may be allocated. */
#define nr_cpumask_bits nr_cpu_ids
static inline void set_nr_cpu_ids(unsigned int nr)
{
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
WARN_ON(nr != nr_cpu_ids);
#else
#define nr_cpumask_bits ((unsigned int)NR_CPUS)
nr_cpu_ids = nr;
#endif
}
/* Deprecated. Always use nr_cpu_ids. */
#define nr_cpumask_bits nr_cpu_ids
/*
* The following particular system cpumasks and operations manage
@ -67,10 +71,6 @@ extern unsigned int nr_cpu_ids;
* cpu_online_mask is the dynamic subset of cpu_present_mask,
* indicating those CPUs available for scheduling.
*
* If HOTPLUG is enabled, then cpu_possible_mask is forced to have
* all NR_CPUS bits set, otherwise it is just the set of CPUs that
* ACPI reports present at boot.
*
* If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
* depending on what ACPI reports as currently plugged in, otherwise
* cpu_present_mask is just a copy of cpu_possible_mask.
@ -174,9 +174,8 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp)
static inline
unsigned int cpumask_next(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
/* n is a prior cpu */
cpumask_check(n + 1);
return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n + 1);
}
@ -189,9 +188,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp)
*/
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
/* n is a prior cpu */
cpumask_check(n + 1);
return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
}
@ -231,9 +229,8 @@ static inline
unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
const struct cpumask *src2p)
{
/* -1 is a legal arg here. */
if (n != -1)
cpumask_check(n);
/* n is a prior cpu */
cpumask_check(n + 1);
return find_next_and_bit(cpumask_bits(src1p), cpumask_bits(src2p),
nr_cpumask_bits, n + 1);
}
@ -246,9 +243,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu(cpu, mask) \
for ((cpu) = -1; \
(cpu) = cpumask_next((cpu), (mask)), \
(cpu) < nr_cpu_ids;)
for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits)
/**
* for_each_cpu_not - iterate over every cpu in a complemented mask
@ -258,17 +253,15 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_not(cpu, mask) \
for ((cpu) = -1; \
(cpu) = cpumask_next_zero((cpu), (mask)), \
(cpu) < nr_cpu_ids;)
for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits)
#if NR_CPUS == 1
static inline
unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
{
cpumask_check(start);
if (n != -1)
cpumask_check(n);
/* n is a prior cpu */
cpumask_check(n + 1);
/*
* Return the first available CPU when wrapping, or when starting before cpu0,
@ -293,10 +286,8 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_wrap(cpu, mask, start) \
for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
(cpu) < nr_cpumask_bits; \
(cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
#define for_each_cpu_wrap(cpu, mask, start) \
for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start)
/**
* for_each_cpu_and - iterate over every cpu in both masks
@ -313,9 +304,25 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_and(cpu, mask1, mask2) \
for ((cpu) = -1; \
(cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
(cpu) < nr_cpu_ids;)
for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits)
/**
* for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
* those present in another.
* @cpu: the (optionally unsigned) integer iterator
* @mask1: the first cpumask pointer
* @mask2: the second cpumask pointer
*
* This saves a temporary CPU mask in many places. It is equivalent to:
* struct cpumask tmp;
* cpumask_andnot(&tmp, &mask1, &mask2);
* for_each_cpu(cpu, &tmp)
* ...
*
* After the loop, cpu is >= nr_cpu_ids.
*/
#define for_each_cpu_andnot(cpu, mask1, mask2) \
for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits)
/**
* cpumask_any_but - return a "random" in a cpumask, but not this one.
@ -337,6 +344,50 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
return i;
}
/**
* cpumask_nth - get the first cpu in a cpumask
* @srcp: the cpumask pointer
* @cpu: the N'th cpu to find, starting from 0
*
* Returns >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
{
return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu));
}
/**
* cpumask_nth_and - get the first cpu in 2 cpumasks
* @srcp1: the cpumask pointer
* @srcp2: the cpumask pointer
* @cpu: the N'th cpu to find, starting from 0
*
* Returns >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline
unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
nr_cpumask_bits, cpumask_check(cpu));
}
/**
* cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd.
* @srcp1: the cpumask pointer
* @srcp2: the cpumask pointer
* @cpu: the N'th cpu to find, starting from 0
*
* Returns >= nr_cpu_ids if such cpu doesn't exist.
*/
static inline
unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2),
nr_cpumask_bits, cpumask_check(cpu));
}
#define CPU_BITS_NONE \
{ \
[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
@ -586,6 +637,17 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp)
return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
}
/**
* cpumask_weight_and - Count of bits in (*srcp1 & *srcp2)
* @srcp1: the cpumask to count bits (< nr_cpu_ids) in.
* @srcp2: the cpumask to count bits (< nr_cpu_ids) in.
*/
static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
const struct cpumask *srcp2)
{
return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits);
}
/**
* cpumask_shift_right - *dstp = *srcp >> n
* @dstp: the cpumask result

310
include/linux/find.h
View File

@ -8,15 +8,33 @@
#include <linux/bitops.h>
extern unsigned long _find_next_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert, unsigned long le);
unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
unsigned long start);
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
unsigned long start);
extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n);
unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n);
unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n);
extern unsigned long _find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size);
extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);
extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size);
#ifdef __BIG_ENDIAN
unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size);
unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned
long size, unsigned long offset);
unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
long size, unsigned long offset);
#endif
#ifndef find_next_bit
/**
* find_next_bit - find the next set bit in a memory region
@ -41,7 +59,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
return val ? __ffs(val) : size;
}
return _find_next_bit(addr, NULL, size, offset, 0UL, 0);
return _find_next_bit(addr, size, offset);
}
#endif
@ -71,7 +89,38 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
return val ? __ffs(val) : size;
}
return _find_next_bit(addr1, addr2, size, offset, 0UL, 0);
return _find_next_and_bit(addr1, addr2, size, offset);
}
#endif
#ifndef find_next_andnot_bit
/**
* find_next_andnot_bit - find the next set bit in *addr1 excluding all the bits
* in *addr2
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_andnot_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size,
unsigned long offset)
{
if (small_const_nbits(size)) {
unsigned long val;
if (unlikely(offset >= size))
return size;
val = *addr1 & ~*addr2 & GENMASK(size - 1, offset);
return val ? __ffs(val) : size;
}
return _find_next_andnot_bit(addr1, addr2, size, offset);
}
#endif
@ -99,7 +148,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
return val == ~0UL ? size : ffz(val);
}
return _find_next_bit(addr, NULL, size, offset, ~0UL, 0);
return _find_next_zero_bit(addr, size, offset);
}
#endif
@ -125,6 +174,87 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
}
#endif
/**
* find_nth_bit - find N'th set bit in a memory region
* @addr: The address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* The following is semantically equivalent:
* idx = find_nth_bit(addr, size, 0);
* idx = find_first_bit(addr, size);
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_bit(addr, size, n);
}
/**
* find_nth_and_bit - find N'th set bit in 2 memory regions
* @addr1: The 1st address to start the search at
* @addr2: The 2nd address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_and_bit(addr1, addr2, size, n);
}
/**
* find_nth_andnot_bit - find N'th set bit in 2 memory regions,
* flipping bits in 2nd region
* @addr1: The 1st address to start the search at
* @addr2: The 2nd address to start the search at
* @size: The maximum number of bits to search
* @n: The number of set bit, which position is needed, counting from 0
*
* Returns the bit number of the N'th set bit.
* If no such, returns @size.
*/
static inline
unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
if (n >= size)
return size;
if (small_const_nbits(size)) {
unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0);
return val ? fns(val, n) : size;
}
return __find_nth_andnot_bit(addr1, addr2, size, n);
}
#ifndef find_first_and_bit
/**
* find_first_and_bit - find the first set bit in both memory regions
@ -193,6 +323,78 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
}
#endif
/**
* find_next_and_bit_wrap - find the next set bit in both memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit, or first set bit up to @offset
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
const unsigned long *addr2,
unsigned long size, unsigned long offset)
{
unsigned long bit = find_next_and_bit(addr1, addr2, size, offset);
if (bit < size)
return bit;
bit = find_first_and_bit(addr1, addr2, offset);
return bit < offset ? bit : size;
}
/**
* find_next_bit_wrap - find the next set bit in both memory regions
* @addr: The first address to base the search on
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit, or first set bit up to @offset
* If no bits are set, returns @size.
*/
static inline
unsigned long find_next_bit_wrap(const unsigned long *addr,
unsigned long size, unsigned long offset)
{
unsigned long bit = find_next_bit(addr, size, offset);
if (bit < size)
return bit;
bit = find_first_bit(addr, offset);
return bit < offset ? bit : size;
}
/*
* Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
* before using it alone.
*/
static inline
unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
unsigned long start, unsigned long n)
{
unsigned long bit;
/* If not wrapped around */
if (n > start) {
/* and have a bit, just return it. */
bit = find_next_bit(bitmap, size, n);
if (bit < size)
return bit;
/* Otherwise, wrap around and ... */
n = 0;
}
/* Search the other part. */
bit = find_next_bit(bitmap, start, n);
return bit < start ? bit : size;
}
/**
* find_next_clump8 - find next 8-bit clump with set bits in a memory region
* @clump: location to store copy of found clump
@ -247,7 +449,21 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned
return val == ~0UL ? size : ffz(val);
}
return _find_next_bit(addr, NULL, size, offset, ~0UL, 1);
return _find_next_zero_bit_le(addr, size, offset);
}
#endif
#ifndef find_first_zero_bit_le
static inline
unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0);
return val == ~0UL ? size : ffz(val);
}
return _find_first_zero_bit_le(addr, size);
}
#endif
@ -266,40 +482,39 @@ unsigned long find_next_bit_le(const void *addr, unsigned
return val ? __ffs(val) : size;
}
return _find_next_bit(addr, NULL, size, offset, 0UL, 1);
return _find_next_bit_le(addr, size, offset);
}
#endif
#ifndef find_first_zero_bit_le
#define find_first_zero_bit_le(addr, size) \
find_next_zero_bit_le((addr), (size), 0)
#endif
#else
#error "Please fix <asm/byteorder.h>"
#endif
#define for_each_set_bit(bit, addr, size) \
for ((bit) = find_next_bit((addr), (size), 0); \
(bit) < (size); \
(bit) = find_next_bit((addr), (size), (bit) + 1))
for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
#define for_each_and_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
#define for_each_andnot_bit(bit, addr1, addr2, size) \
for ((bit) = 0; \
(bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
(bit)++)
/* same as for_each_set_bit() but use bit as value to start with */
#define for_each_set_bit_from(bit, addr, size) \
for ((bit) = find_next_bit((addr), (size), (bit)); \
(bit) < (size); \
(bit) = find_next_bit((addr), (size), (bit) + 1))
for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
#define for_each_clear_bit(bit, addr, size) \
for ((bit) = find_next_zero_bit((addr), (size), 0); \
(bit) < (size); \
(bit) = find_next_zero_bit((addr), (size), (bit) + 1))
for ((bit) = 0; \
(bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \
(bit)++)
/* same as for_each_clear_bit() but use bit as value to start with */
#define for_each_clear_bit_from(bit, addr, size) \
for ((bit) = find_next_zero_bit((addr), (size), (bit)); \
(bit) < (size); \
(bit) = find_next_zero_bit((addr), (size), (bit) + 1))
for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++)
/**
* for_each_set_bitrange - iterate over all set bit ranges [b; e)
@ -309,11 +524,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
* @size: bitmap size in number of bits
*/
#define for_each_set_bitrange(b, e, addr, size) \
for ((b) = find_next_bit((addr), (size), 0), \
(e) = find_next_zero_bit((addr), (size), (b) + 1); \
for ((b) = 0; \
(b) = find_next_bit((addr), (size), b), \
(e) = find_next_zero_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = find_next_bit((addr), (size), (e) + 1), \
(e) = find_next_zero_bit((addr), (size), (b) + 1))
(b) = (e) + 1)
/**
* for_each_set_bitrange_from - iterate over all set bit ranges [b; e)
@ -323,11 +538,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
* @size: bitmap size in number of bits
*/
#define for_each_set_bitrange_from(b, e, addr, size) \
for ((b) = find_next_bit((addr), (size), (b)), \
(e) = find_next_zero_bit((addr), (size), (b) + 1); \
for (; \
(b) = find_next_bit((addr), (size), (b)), \
(e) = find_next_zero_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = find_next_bit((addr), (size), (e) + 1), \
(e) = find_next_zero_bit((addr), (size), (b) + 1))
(b) = (e) + 1)
/**
* for_each_clear_bitrange - iterate over all unset bit ranges [b; e)
@ -337,11 +552,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
* @size: bitmap size in number of bits
*/
#define for_each_clear_bitrange(b, e, addr, size) \
for ((b) = find_next_zero_bit((addr), (size), 0), \
(e) = find_next_bit((addr), (size), (b) + 1); \
for ((b) = 0; \
(b) = find_next_zero_bit((addr), (size), (b)), \
(e) = find_next_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = find_next_zero_bit((addr), (size), (e) + 1), \
(e) = find_next_bit((addr), (size), (b) + 1))
(b) = (e) + 1)
/**
* for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e)
@ -351,11 +566,24 @@ unsigned long find_next_bit_le(const void *addr, unsigned
* @size: bitmap size in number of bits
*/
#define for_each_clear_bitrange_from(b, e, addr, size) \
for ((b) = find_next_zero_bit((addr), (size), (b)), \
(e) = find_next_bit((addr), (size), (b) + 1); \
for (; \
(b) = find_next_zero_bit((addr), (size), (b)), \
(e) = find_next_bit((addr), (size), (b) + 1), \
(b) < (size); \
(b) = find_next_zero_bit((addr), (size), (e) + 1), \
(e) = find_next_bit((addr), (size), (b) + 1))
(b) = (e) + 1)
/**
* for_each_set_bit_wrap - iterate over all set bits starting from @start, and
* wrapping around the end of bitmap.
* @bit: offset for current iteration
* @addr: bitmap address to base the search on
* @size: bitmap size in number of bits
* @start: Starting bit for bitmap traversing, wrapping around the bitmap end
*/
#define for_each_set_bit_wrap(bit, addr, size, start) \
for ((bit) = find_next_bit_wrap((addr), (size), (start)); \
(bit) < (size); \
(bit) = __for_each_wrap((addr), (size), (start), (bit) + 1))
/**
* for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits

10
include/linux/netdevice.h
View File

@ -3663,9 +3663,8 @@ static inline bool netif_attr_test_online(unsigned long j,
static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
unsigned int nr_bits)
{
/* -1 is a legal arg here. */
if (n != -1)
cpu_max_bits_warn(n, nr_bits);
/* n is a prior cpu */
cpu_max_bits_warn(n + 1, nr_bits);
if (srcp)
return find_next_bit(srcp, nr_bits, n + 1);
@ -3686,9 +3685,8 @@ static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
const unsigned long *src2p,
unsigned int nr_bits)
{
/* -1 is a legal arg here. */
if (n != -1)
cpu_max_bits_warn(n, nr_bits);
/* n is a prior cpu */
cpu_max_bits_warn(n + 1, nr_bits);
if (src1p && src2p)
return find_next_and_bit(src1p, src2p, nr_bits, n + 1);

3
include/linux/nodemask.h
View File

@ -508,8 +508,7 @@ static inline int node_random(const nodemask_t *maskp)
w = nodes_weight(*maskp);
if (w)
bit = bitmap_ord_to_pos(maskp->bits,
get_random_int() % w, MAX_NUMNODES);
bit = find_nth_bit(maskp->bits, MAX_NUMNODES, get_random_int() % w);
return bit;
#else
return 0;

5
kernel/sched/core.c
View File

@ -357,10 +357,7 @@ static void __sched_core_flip(bool enabled)
/*
* Toggle the offline CPUs.
*/
cpumask_copy(&sched_core_mask, cpu_possible_mask);
cpumask_andnot(&sched_core_mask, &sched_core_mask, cpu_online_mask);
for_each_cpu(cpu, &sched_core_mask)
for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask)
cpu_rq(cpu)->core_enabled = enabled;
cpus_read_unlock();

6
kernel/smp.c
View File

@ -1069,7 +1069,7 @@ static int __init nrcpus(char *str)
int nr_cpus;
if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids)
nr_cpu_ids = nr_cpus;
set_nr_cpu_ids(nr_cpus);
return 0;
}
@ -1087,14 +1087,16 @@ static int __init maxcpus(char *str)
early_param("maxcpus", maxcpus);
#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS)
/* Setup number of possible processor ids */
unsigned int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);
#endif
/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
void __init setup_nr_cpu_ids(void)
{
nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
}
/* Called by boot processor to activate the rest. */

9
lib/Kconfig
View File

@ -531,6 +531,15 @@ config CPUMASK_OFFSTACK
them on the stack. This is a bit more expensive, but avoids
stack overflow.
config FORCE_NR_CPUS
bool "NR_CPUS is set to an actual number of CPUs"
depends on SMP
help
Say Yes if you have NR_CPUS set to an actual number of possible
CPUs in your system, not to a default value. This forces the core
code to rely on compile-time value and optimize kernel routines
better.
config CPU_RMAP
bool
depends on SMP

68
lib/bitmap.c
View File

@ -333,20 +333,32 @@ bool __bitmap_subset(const unsigned long *bitmap1,
}
EXPORT_SYMBOL(__bitmap_subset);
#define BITMAP_WEIGHT(FETCH, bits) \
({ \
unsigned int __bits = (bits), idx, w = 0; \
\
for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \
w += hweight_long(FETCH); \
\
if (__bits % BITS_PER_LONG) \
w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \
\
w; \
})
unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits)
{
unsigned int k, lim = bits/BITS_PER_LONG, w = 0;
for (k = 0; k < lim; k++)
w += hweight_long(bitmap[k]);
if (bits % BITS_PER_LONG)
w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits));
return w;
return BITMAP_WEIGHT(bitmap[idx], bits);
}
EXPORT_SYMBOL(__bitmap_weight);
unsigned int __bitmap_weight_and(const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits);
}
EXPORT_SYMBOL(__bitmap_weight_and);
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
@ -953,37 +965,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne
if (pos >= nbits || !test_bit(pos, buf))
return -1;
return __bitmap_weight(buf, pos);
}
/**
* bitmap_ord_to_pos - find position of n-th set bit in bitmap
* @buf: pointer to bitmap
* @ord: ordinal bit position (n-th set bit, n >= 0)
* @nbits: number of valid bit positions in @buf
*
* Map the ordinal offset of bit @ord in @buf to its position in @buf.
* Value of @ord should be in range 0 <= @ord < weight(buf). If @ord
* >= weight(buf), returns @nbits.
*
* If for example, just bits 4 through 7 are set in @buf, then @ord
* values 0 through 3 will get mapped to 4 through 7, respectively,
* and all other @ord values returns @nbits. When @ord value 3
* gets mapped to (returns) @pos value 7 in this example, that means
* that the 3rd set bit (starting with 0th) is at position 7 in @buf.
*
* The bit positions 0 through @nbits-1 are valid positions in @buf.
*/
unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits)
{
unsigned int pos;
for (pos = find_first_bit(buf, nbits);
pos < nbits && ord;
pos = find_next_bit(buf, nbits, pos + 1))
ord--;
return pos;
return bitmap_weight(buf, pos);
}
/**
@ -1035,7 +1017,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src,
if (n < 0 || w == 0)
set_bit(oldbit, dst); /* identity map */
else
set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst);
set_bit(find_nth_bit(new, nbits, n % w), dst);
}
}
EXPORT_SYMBOL(bitmap_remap);
@ -1074,7 +1056,7 @@ int bitmap_bitremap(int oldbit, const unsigned long *old,
if (n < 0 || w == 0)
return oldbit;
else
return bitmap_ord_to_pos(new, n % w, bits);
return find_nth_bit(new, bits, n % w);
}
EXPORT_SYMBOL(bitmap_bitremap);
@ -1198,7 +1180,7 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig,
* The following code is a more efficient, but less
* obvious, equivalent to the loop:
* for (m = 0; m < bitmap_weight(relmap, bits); m++) {
* n = bitmap_ord_to_pos(orig, m, bits);
* n = find_nth_bit(orig, bits, m);
* if (test_bit(m, orig))
* set_bit(n, dst);
* }

36
lib/cpumask.c
View File

@ -128,23 +128,21 @@ unsigned int cpumask_local_spread(unsigned int i, int node)
i %= num_online_cpus();
if (node == NUMA_NO_NODE) {
for_each_cpu(cpu, cpu_online_mask)
if (i-- == 0)
return cpu;
cpu = cpumask_nth(i, cpu_online_mask);
if (cpu < nr_cpu_ids)
return cpu;
} else {
/* NUMA first. */
for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
if (i-- == 0)
return cpu;
cpu = cpumask_nth_and(i, cpu_online_mask, cpumask_of_node(node));
if (cpu < nr_cpu_ids)
return cpu;
for_each_cpu(cpu, cpu_online_mask) {
/* Skip NUMA nodes, done above. */
if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
continue;
i -= cpumask_weight_and(cpu_online_mask, cpumask_of_node(node));
if (i-- == 0)
return cpu;
}
/* Skip NUMA nodes, done above. */
cpu = cpumask_nth_andnot(i, cpu_online_mask, cpumask_of_node(node));
if (cpu < nr_cpu_ids)
return cpu;
}
BUG();
}
@ -168,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
/* NOTE: our first selection will skip 0. */
prev = __this_cpu_read(distribute_cpu_mask_prev);
next = cpumask_next_and(prev, src1p, src2p);
if (next >= nr_cpu_ids)
next = cpumask_first_and(src1p, src2p);
next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),
nr_cpumask_bits, prev + 1);
if (next < nr_cpu_ids)
__this_cpu_write(distribute_cpu_mask_prev, next);
@ -185,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp)
/* NOTE: our first selection will skip 0. */
prev = __this_cpu_read(distribute_cpu_mask_prev);
next = cpumask_next(prev, srcp);
if (next >= nr_cpu_ids)
next = cpumask_first(srcp);
next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1);
if (next < nr_cpu_ids)
__this_cpu_write(distribute_cpu_mask_prev, next);

19
lib/cpumask_kunit.c
View File

@ -33,6 +33,19 @@
KUNIT_EXPECT_EQ_MSG((test), nr_cpu_ids - mask_weight, iter, MASK_MSG(mask)); \
} while (0)
#define EXPECT_FOR_EACH_CPU_OP_EQ(test, op, mask1, mask2) \
do { \
const cpumask_t *m1 = (mask1); \
const cpumask_t *m2 = (mask2); \
int weight; \
int cpu, iter = 0; \
cpumask_##op(&mask_tmp, m1, m2); \
weight = cpumask_weight(&mask_tmp); \
for_each_cpu_##op(cpu, mask1, mask2) \
iter++; \
KUNIT_EXPECT_EQ((test), weight, iter); \
} while (0)
#define EXPECT_FOR_EACH_CPU_WRAP_EQ(test, mask) \
do { \
const cpumask_t *m = (mask); \
@ -54,6 +67,7 @@
static cpumask_t mask_empty;
static cpumask_t mask_all;
static cpumask_t mask_tmp;
static void test_cpumask_weight(struct kunit *test)
{
@ -101,10 +115,15 @@ static void test_cpumask_iterators(struct kunit *test)
EXPECT_FOR_EACH_CPU_EQ(test, &mask_empty);
EXPECT_FOR_EACH_CPU_NOT_EQ(test, &mask_empty);
EXPECT_FOR_EACH_CPU_WRAP_EQ(test, &mask_empty);
EXPECT_FOR_EACH_CPU_OP_EQ(test, and, &mask_empty, &mask_empty);
EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, &mask_empty);
EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, &mask_empty, &mask_empty);
EXPECT_FOR_EACH_CPU_EQ(test, cpu_possible_mask);
EXPECT_FOR_EACH_CPU_NOT_EQ(test, cpu_possible_mask);
EXPECT_FOR_EACH_CPU_WRAP_EQ(test, cpu_possible_mask);
EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, cpu_possible_mask);
EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, cpu_possible_mask, &mask_empty);
}
static void test_cpumask_iterators_builtin(struct kunit *test)

233
lib/find_bit.c
View File

@ -19,57 +19,78 @@
#include <linux/minmax.h>
#include <linux/swab.h>
#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \
!defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \
!defined(find_next_and_bit)
/*
* This is a common helper function for find_next_bit, find_next_zero_bit, and
* find_next_and_bit. The differences are:
* - The "invert" argument, which is XORed with each fetched word before
* searching it for one bits.
* - The optional "addr2", which is anded with "addr1" if present.
* Common helper for find_bit() function family
* @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
* @MUNGE: The expression that post-processes a word containing found bit (may be empty)
* @size: The bitmap size in bits
*/
unsigned long _find_next_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert, unsigned long le)
{
unsigned long tmp, mask;
#define FIND_FIRST_BIT(FETCH, MUNGE, size) \
({ \
unsigned long idx, val, sz = (size); \
\
for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \
val = (FETCH); \
if (val) { \
sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \
break; \
} \
} \
\
sz; \
})
if (unlikely(start >= nbits))
return nbits;
/*
* Common helper for find_next_bit() function family
* @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
* @MUNGE: The expression that post-processes a word containing found bit (may be empty)
* @size: The bitmap size in bits
* @start: The bitnumber to start searching at
*/
#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \
({ \
unsigned long mask, idx, tmp, sz = (size), __start = (start); \
\
if (unlikely(__start >= sz)) \
goto out; \
\
mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \
idx = __start / BITS_PER_LONG; \
\
for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \
if ((idx + 1) * BITS_PER_LONG >= sz) \
goto out; \
idx++; \
} \
\
sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \
out: \
sz; \
})
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
/* Handle 1st word. */
mask = BITMAP_FIRST_WORD_MASK(start);
if (le)
mask = swab(mask);
tmp &= mask;
start = round_down(start, BITS_PER_LONG);
while (!tmp) {
start += BITS_PER_LONG;
if (start >= nbits)
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
}
if (le)
tmp = swab(tmp);
return min(start + __ffs(tmp), nbits);
}
EXPORT_SYMBOL(_find_next_bit);
#endif
#define FIND_NTH_BIT(FETCH, size, num) \
({ \
unsigned long sz = (size), nr = (num), idx, w, tmp; \
\
for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) { \
if (idx * BITS_PER_LONG + nr >= sz) \
goto out; \
\
tmp = (FETCH); \
w = hweight_long(tmp); \
if (w > nr) \
goto found; \
\
nr -= w; \
} \
\
if (sz % BITS_PER_LONG) \
tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz); \
found: \
sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz); \
out: \
sz; \
})
#ifndef find_first_bit
/*
@ -77,14 +98,7 @@ EXPORT_SYMBOL(_find_next_bit);
*/
unsigned long _find_first_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx])
return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size);
}
return size;
return FIND_FIRST_BIT(addr[idx], /* nop */, size);
}
EXPORT_SYMBOL(_find_first_bit);
#endif
@ -97,15 +111,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2,
unsigned long size)
{
unsigned long idx, val;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
val = addr1[idx] & addr2[idx];
if (val)
return min(idx * BITS_PER_LONG + __ffs(val), size);
}
return size;
return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size);
}
EXPORT_SYMBOL(_find_first_and_bit);
#endif
@ -116,18 +122,66 @@ EXPORT_SYMBOL(_find_first_and_bit);
*/
unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx] != ~0UL)
return min(idx * BITS_PER_LONG + ffz(addr[idx]), size);
}
return size;
return FIND_FIRST_BIT(~addr[idx], /* nop */, size);
}
EXPORT_SYMBOL(_find_first_zero_bit);
#endif
#ifndef find_next_bit
unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start)
{
return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start);
}
EXPORT_SYMBOL(_find_next_bit);
#endif
unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
{
return FIND_NTH_BIT(addr[idx], size, n);
}
EXPORT_SYMBOL(__find_nth_bit);
unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n);
}
EXPORT_SYMBOL(__find_nth_and_bit);
unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long size, unsigned long n)
{
return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n);
}
EXPORT_SYMBOL(__find_nth_andnot_bit);
#ifndef find_next_and_bit
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start)
{
return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start);
}
EXPORT_SYMBOL(_find_next_and_bit);
#endif
#ifndef find_next_andnot_bit
unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start)
{
return FIND_NEXT_BIT(addr1[idx] & ~addr2[idx], /* nop */, nbits, start);
}
EXPORT_SYMBOL(_find_next_andnot_bit);
#endif
#ifndef find_next_zero_bit
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
unsigned long start)
{
return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start);
}
EXPORT_SYMBOL(_find_next_zero_bit);
#endif
#ifndef find_last_bit
unsigned long _find_last_bit(const unsigned long *addr, unsigned long size)
{
@ -161,3 +215,38 @@ unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr,
return offset;
}
EXPORT_SYMBOL(find_next_clump8);
#ifdef __BIG_ENDIAN
#ifndef find_first_zero_bit_le
/*
* Find the first cleared bit in an LE memory region.
*/
unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size)
{
return FIND_FIRST_BIT(~addr[idx], swab, size);
}
EXPORT_SYMBOL(_find_first_zero_bit_le);
#endif
#ifndef find_next_zero_bit_le
unsigned long _find_next_zero_bit_le(const unsigned long *addr,
unsigned long size, unsigned long offset)
{
return FIND_NEXT_BIT(~addr[idx], swab, size, offset);
}
EXPORT_SYMBOL(_find_next_zero_bit_le);
#endif
#ifndef find_next_bit_le
unsigned long _find_next_bit_le(const unsigned long *addr,
unsigned long size, unsigned long offset)
{
return FIND_NEXT_BIT(addr[idx], swab, size, offset);
}
EXPORT_SYMBOL(_find_next_bit_le);
#endif
#endif /* __BIG_ENDIAN */

18
lib/find_bit_benchmark.c
View File

@ -115,6 +115,22 @@ static int __init test_find_last_bit(const void *bitmap, unsigned long len)
return 0;
}
static int __init test_find_nth_bit(const unsigned long *bitmap, unsigned long len)
{
unsigned long l, n, w = bitmap_weight(bitmap, len);
ktime_t time;
time = ktime_get();
for (n = 0; n < w; n++) {
l = find_nth_bit(bitmap, len, n);
WARN_ON(l >= len);
}
time = ktime_get() - time;
pr_err("find_nth_bit: %18llu ns, %6ld iterations\n", time, w);
return 0;
}
static int __init test_find_next_and_bit(const void *bitmap,
const void *bitmap2, unsigned long len)
{
@ -142,6 +158,7 @@ static int __init find_bit_test(void)
test_find_next_bit(bitmap, BITMAP_LEN);
test_find_next_zero_bit(bitmap, BITMAP_LEN);
test_find_last_bit(bitmap, BITMAP_LEN);
test_find_nth_bit(bitmap, BITMAP_LEN / 10);
/*
* test_find_first_bit() may take some time, so
@ -164,6 +181,7 @@ static int __init find_bit_test(void)
test_find_next_bit(bitmap, BITMAP_LEN);
test_find_next_zero_bit(bitmap, BITMAP_LEN);
test_find_last_bit(bitmap, BITMAP_LEN);
test_find_nth_bit(bitmap, BITMAP_LEN);
test_find_first_bit(bitmap, BITMAP_LEN);
test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN);
test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);

291
lib/test_bitmap.c
View File

@ -16,6 +16,8 @@
#include "../tools/testing/selftests/kselftest_module.h"
#define EXP1_IN_BITS (sizeof(exp1) * 8)
KSTM_MODULE_GLOBALS();
static char pbl_buffer[PAGE_SIZE] __initdata;
@ -219,6 +221,47 @@ static void __init test_zero_clear(void)
expect_eq_pbl("", bmap, 1024);
}
static void __init test_find_nth_bit(void)
{
unsigned long b, bit, cnt = 0;
DECLARE_BITMAP(bmap, 64 * 3);
bitmap_zero(bmap, 64 * 3);
__set_bit(10, bmap);
__set_bit(20, bmap);
__set_bit(30, bmap);
__set_bit(40, bmap);
__set_bit(50, bmap);
__set_bit(60, bmap);
__set_bit(80, bmap);
__set_bit(123, bmap);
expect_eq_uint(10, find_nth_bit(bmap, 64 * 3, 0));
expect_eq_uint(20, find_nth_bit(bmap, 64 * 3, 1));
expect_eq_uint(30, find_nth_bit(bmap, 64 * 3, 2));
expect_eq_uint(40, find_nth_bit(bmap, 64 * 3, 3));
expect_eq_uint(50, find_nth_bit(bmap, 64 * 3, 4));
expect_eq_uint(60, find_nth_bit(bmap, 64 * 3, 5));
expect_eq_uint(80, find_nth_bit(bmap, 64 * 3, 6));
expect_eq_uint(123, find_nth_bit(bmap, 64 * 3, 7));
expect_eq_uint(64 * 3, find_nth_bit(bmap, 64 * 3, 8));
expect_eq_uint(10, find_nth_bit(bmap, 64 * 3 - 1, 0));
expect_eq_uint(20, find_nth_bit(bmap, 64 * 3 - 1, 1));
expect_eq_uint(30, find_nth_bit(bmap, 64 * 3 - 1, 2));
expect_eq_uint(40, find_nth_bit(bmap, 64 * 3 - 1, 3));
expect_eq_uint(50, find_nth_bit(bmap, 64 * 3 - 1, 4));
expect_eq_uint(60, find_nth_bit(bmap, 64 * 3 - 1, 5));
expect_eq_uint(80, find_nth_bit(bmap, 64 * 3 - 1, 6));
expect_eq_uint(123, find_nth_bit(bmap, 64 * 3 - 1, 7));
expect_eq_uint(64 * 3 - 1, find_nth_bit(bmap, 64 * 3 - 1, 8));
for_each_set_bit(bit, exp1, EXP1_IN_BITS) {
b = find_nth_bit(exp1, EXP1_IN_BITS, cnt++);
expect_eq_uint(b, bit);
}
}
static void __init test_fill_set(void)
{
DECLARE_BITMAP(bmap, 1024);
@ -557,8 +600,6 @@ static void __init test_bitmap_parse(void)
}
}
#define EXP1_IN_BITS (sizeof(exp1) * 8)
static void __init test_bitmap_arr32(void)
{
unsigned int nbits, next_bit;
@ -685,6 +726,239 @@ static void __init test_for_each_set_clump8(void)
expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump);
}
static void __init test_for_each_set_bit_wrap(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int wr, bit;
bitmap_zero(orig, 500);
/* Set individual bits */
for (bit = 0; bit < 500; bit += 10)
bitmap_set(orig, bit, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for (wr = 0; wr < 500; wr++) {
bitmap_zero(copy, 500);
for_each_set_bit_wrap(bit, orig, 500, wr)
bitmap_set(copy, bit, 1);
expect_eq_bitmap(orig, copy, 500);
}
}
static void __init test_for_each_set_bit(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int bit;
bitmap_zero(orig, 500);
bitmap_zero(copy, 500);
/* Set individual bits */
for (bit = 0; bit < 500; bit += 10)
bitmap_set(orig, bit, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for_each_set_bit(bit, orig, 500)
bitmap_set(copy, bit, 1);
expect_eq_bitmap(orig, copy, 500);
}
static void __init test_for_each_set_bit_from(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int wr, bit;
bitmap_zero(orig, 500);
/* Set individual bits */
for (bit = 0; bit < 500; bit += 10)
bitmap_set(orig, bit, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for (wr = 0; wr < 500; wr++) {
DECLARE_BITMAP(tmp, 500);
bitmap_zero(copy, 500);
bit = wr;
for_each_set_bit_from(bit, orig, 500)
bitmap_set(copy, bit, 1);
bitmap_copy(tmp, orig, 500);
bitmap_clear(tmp, 0, wr);
expect_eq_bitmap(tmp, copy, 500);
}
}
static void __init test_for_each_clear_bit(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int bit;
bitmap_fill(orig, 500);
bitmap_fill(copy, 500);
/* Set individual bits */
for (bit = 0; bit < 500; bit += 10)
bitmap_clear(orig, bit, 1);
/* Set range of bits */
bitmap_clear(orig, 100, 50);
for_each_clear_bit(bit, orig, 500)
bitmap_clear(copy, bit, 1);
expect_eq_bitmap(orig, copy, 500);
}
static void __init test_for_each_clear_bit_from(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int wr, bit;
bitmap_fill(orig, 500);
/* Set individual bits */
for (bit = 0; bit < 500; bit += 10)
bitmap_clear(orig, bit, 1);
/* Set range of bits */
bitmap_clear(orig, 100, 50);
for (wr = 0; wr < 500; wr++) {
DECLARE_BITMAP(tmp, 500);
bitmap_fill(copy, 500);
bit = wr;
for_each_clear_bit_from(bit, orig, 500)
bitmap_clear(copy, bit, 1);
bitmap_copy(tmp, orig, 500);
bitmap_set(tmp, 0, wr);
expect_eq_bitmap(tmp, copy, 500);
}
}
static void __init test_for_each_set_bitrange(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int s, e;
bitmap_zero(orig, 500);
bitmap_zero(copy, 500);
/* Set individual bits */
for (s = 0; s < 500; s += 10)
bitmap_set(orig, s, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for_each_set_bitrange(s, e, orig, 500)
bitmap_set(copy, s, e-s);
expect_eq_bitmap(orig, copy, 500);
}
static void __init test_for_each_clear_bitrange(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int s, e;
bitmap_fill(orig, 500);
bitmap_fill(copy, 500);
/* Set individual bits */
for (s = 0; s < 500; s += 10)
bitmap_clear(orig, s, 1);
/* Set range of bits */
bitmap_clear(orig, 100, 50);
for_each_clear_bitrange(s, e, orig, 500)
bitmap_clear(copy, s, e-s);
expect_eq_bitmap(orig, copy, 500);
}
static void __init test_for_each_set_bitrange_from(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int wr, s, e;
bitmap_zero(orig, 500);
/* Set individual bits */
for (s = 0; s < 500; s += 10)
bitmap_set(orig, s, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for (wr = 0; wr < 500; wr++) {
DECLARE_BITMAP(tmp, 500);
bitmap_zero(copy, 500);
s = wr;
for_each_set_bitrange_from(s, e, orig, 500)
bitmap_set(copy, s, e - s);
bitmap_copy(tmp, orig, 500);
bitmap_clear(tmp, 0, wr);
expect_eq_bitmap(tmp, copy, 500);
}
}
static void __init test_for_each_clear_bitrange_from(void)
{
DECLARE_BITMAP(orig, 500);
DECLARE_BITMAP(copy, 500);
unsigned int wr, s, e;
bitmap_fill(orig, 500);
/* Set individual bits */
for (s = 0; s < 500; s += 10)
bitmap_clear(orig, s, 1);
/* Set range of bits */
bitmap_set(orig, 100, 50);
for (wr = 0; wr < 500; wr++) {
DECLARE_BITMAP(tmp, 500);
bitmap_fill(copy, 500);
s = wr;
for_each_clear_bitrange_from(s, e, orig, 500)
bitmap_clear(copy, s, e - s);
bitmap_copy(tmp, orig, 500);
bitmap_set(tmp, 0, wr);
expect_eq_bitmap(tmp, copy, 500);
}
}
struct test_bitmap_cut {
unsigned int first;
unsigned int cut;
@ -948,10 +1222,21 @@ static void __init selftest(void)
test_bitmap_parselist();
test_bitmap_printlist();
test_mem_optimisations();
test_for_each_set_clump8();
test_bitmap_cut();
test_bitmap_print_buf();
test_bitmap_const_eval();
test_find_nth_bit();
test_for_each_set_bit();
test_for_each_set_bit_from();
test_for_each_clear_bit();
test_for_each_clear_bit_from();
test_for_each_set_bitrange();
test_for_each_clear_bitrange();
test_for_each_set_bitrange_from();
test_for_each_clear_bitrange_from();
test_for_each_set_clump8();
test_for_each_set_bit_wrap();
}
KSTM_MODULE_LOADERS(test_bitmap);

61
tools/include/linux/find.h
View File

@ -8,21 +8,23 @@
#include <linux/bitops.h>
extern unsigned long _find_next_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert, unsigned long le);
unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
unsigned long start);
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start);
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
unsigned long start);
extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
extern unsigned long _find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long size);
extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);
extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size);
#ifndef find_next_bit
/**
* find_next_bit - find the next set bit in a memory region
* @addr: The address to base the search on
* @offset: The bitnumber to start searching at
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
@ -41,7 +43,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
return val ? __ffs(val) : size;
}
return _find_next_bit(addr, NULL, size, offset, 0UL, 0);
return _find_next_bit(addr, size, offset);
}
#endif
@ -50,8 +52,8 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
* find_next_and_bit - find the next set bit in both memory regions
* @addr1: The first address to base the search on
* @addr2: The second address to base the search on
* @offset: The bitnumber to start searching at
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number for the next set bit
* If no bits are set, returns @size.
@ -71,7 +73,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
return val ? __ffs(val) : size;
}
return _find_next_bit(addr1, addr2, size, offset, 0UL, 0);
return _find_next_and_bit(addr1, addr2, size, offset);
}
#endif
@ -79,8 +81,8 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
/**
* find_next_zero_bit - find the next cleared bit in a memory region
* @addr: The address to base the search on
* @offset: The bitnumber to start searching at
* @size: The bitmap size in bits
* @offset: The bitnumber to start searching at
*
* Returns the bit number of the next zero bit
* If no bits are zero, returns @size.
@ -99,7 +101,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
return val == ~0UL ? size : ffz(val);
}
return _find_next_bit(addr, NULL, size, offset, ~0UL, 0);
return _find_next_zero_bit(addr, size, offset);
}
#endif
@ -172,43 +174,4 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
}
#endif
#ifndef find_last_bit
/**
* find_last_bit - find the last set bit in a memory region
* @addr: The address to start the search at
* @size: The number of bits to search
*
* Returns the bit number of the last set bit, or size.
*/
static inline
unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
{
if (small_const_nbits(size)) {
unsigned long val = *addr & GENMASK(size - 1, 0);
return val ? __fls(val) : size;
}
return _find_last_bit(addr, size);
}
#endif
/**
* find_next_clump8 - find next 8-bit clump with set bits in a memory region
* @clump: location to store copy of found clump
* @addr: address to base the search on
* @size: bitmap size in number of bits
* @offset: bit offset at which to start searching
*
* Returns the bit offset for the next set clump; the found clump value is
* copied to the location pointed by @clump. If no bits are set, returns @size.
*/
extern unsigned long find_next_clump8(unsigned long *clump,
const unsigned long *addr,
unsigned long size, unsigned long offset);
#define find_first_clump8(clump, bits, size) \
find_next_clump8((clump), (bits), (size), 0)
#endif /*__LINUX_FIND_H_ */

153
tools/lib/find_bit.c
View File

@ -18,66 +18,54 @@
#include <linux/bitmap.h>
#include <linux/kernel.h>
#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \
!defined(find_next_and_bit)
/*
* Common helper for find_bit() function family
* @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
* @MUNGE: The expression that post-processes a word containing found bit (may be empty)
* @size: The bitmap size in bits
*/
#define FIND_FIRST_BIT(FETCH, MUNGE, size) \
({ \
unsigned long idx, val, sz = (size); \
\
for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \
val = (FETCH); \
if (val) { \
sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \
break; \
} \
} \
\
sz; \
})
/*
* This is a common helper function for find_next_bit, find_next_zero_bit, and
* find_next_and_bit. The differences are:
* - The "invert" argument, which is XORed with each fetched word before
* searching it for one bits.
* - The optional "addr2", which is anded with "addr1" if present.
* Common helper for find_next_bit() function family
* @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
* @MUNGE: The expression that post-processes a word containing found bit (may be empty)
* @size: The bitmap size in bits
* @start: The bitnumber to start searching at
*/
unsigned long _find_next_bit(const unsigned long *addr1,
const unsigned long *addr2, unsigned long nbits,
unsigned long start, unsigned long invert, unsigned long le)
{
unsigned long tmp, mask;
(void) le;
if (unlikely(start >= nbits))
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
/* Handle 1st word. */
mask = BITMAP_FIRST_WORD_MASK(start);
/*
* Due to the lack of swab() in tools, and the fact that it doesn't
* need little-endian support, just comment it out
*/
#if (0)
if (le)
mask = swab(mask);
#endif
tmp &= mask;
start = round_down(start, BITS_PER_LONG);
while (!tmp) {
start += BITS_PER_LONG;
if (start >= nbits)
return nbits;
tmp = addr1[start / BITS_PER_LONG];
if (addr2)
tmp &= addr2[start / BITS_PER_LONG];
tmp ^= invert;
}
#if (0)
if (le)
tmp = swab(tmp);
#endif
return min(start + __ffs(tmp), nbits);
}
#endif
#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \
({ \
unsigned long mask, idx, tmp, sz = (size), __start = (start); \
\
if (unlikely(__start >= sz)) \
goto out; \
\
mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \
idx = __start / BITS_PER_LONG; \
\
for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \
if ((idx + 1) * BITS_PER_LONG >= sz) \
goto out; \
idx++; \
} \
\
sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \
out: \
sz; \
})
#ifndef find_first_bit
/*
@ -85,14 +73,7 @@ unsigned long _find_next_bit(const unsigned long *addr1,
*/
unsigned long _find_first_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx])
return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size);
}
return size;
return FIND_FIRST_BIT(addr[idx], /* nop */, size);
}
#endif
@ -104,15 +85,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1,
const unsigned long *addr2,
unsigned long size)
{
unsigned long idx, val;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
val = addr1[idx] & addr2[idx];
if (val)
return min(idx * BITS_PER_LONG + __ffs(val), size);
}
return size;
return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size);
}
#endif
@ -122,13 +95,29 @@ unsigned long _find_first_and_bit(const unsigned long *addr1,
*/
unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size)
{
unsigned long idx;
for (idx = 0; idx * BITS_PER_LONG < size; idx++) {
if (addr[idx] != ~0UL)
return min(idx * BITS_PER_LONG + ffz(addr[idx]), size);
}
return size;
return FIND_FIRST_BIT(~addr[idx], /* nop */, size);
}
#endif
#ifndef find_next_bit
unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start)
{
return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start);
}
#endif
#ifndef find_next_and_bit
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
unsigned long nbits, unsigned long start)
{
return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start);
}
#endif
#ifndef find_next_zero_bit
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
unsigned long start)
{
return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start);
}
#endif