lib/find_bit: optimize find_next_bit() functions
Over the past couple years, the function _find_next_bit() was extended
with parameters that modify its behavior to implement and- zero- and le-
flavors. The parameters are passed at compile time, but current design
prevents a compiler from optimizing out the conditionals.
As find_next_bit() API grows, I expect that more parameters will be added.
Current design would require more conditional code in _find_next_bit(),
which would bloat the helper even more and make it barely readable.
This patch replaces _find_next_bit() with a macro FIND_NEXT_BIT, and adds
a set of wrappers, so that the compile-time optimizations become possible.
The common logic is moved to the new macro, and all flavors may be
generated by providing a FETCH macro parameter, like in this example:
#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) ...
find_next_xornot_and_bit(addr1, addr2, addr3, size, start)
{
return FIND_NEXT_BIT(addr1[idx] ^ ~addr2[idx] & addr3[idx],
/* nop */, size, start);
}
The FETCH may be of any complexity, as soon as it only refers the bitmap(s)
and an iterator idx.
MUNGE is here to support _le code generation for BE builds. May be
empty.
I ran find_bit_benchmark 16 times on top of 6.0-rc2 and 16 times on top
of 6.0-rc2 + this series. The results for kvm/x86_64 are:
v6.0-rc2 Optimized Difference Z-score
Random dense bitmap ns ns ns %
find_next_bit: 787735 670546 117189 14.9 3.97
find_next_zero_bit: 777492 664208 113284 14.6 10.51
find_last_bit: 830925 687573 143352 17.3 2.35
find_first_bit: 3874366 3306635 567731 14.7 1.84
find_first_and_bit: 40677125 37739887 2937238 7.2 1.36
find_next_and_bit: 347865 304456 43409 12.5 1.35
Random sparse bitmap
find_next_bit: 19816 14021 5795 29.2 6.10
find_next_zero_bit: 1318901 1223794 95107 7.2 1.41
find_last_bit: 14573 13514 1059 7.3 6.92
find_first_bit: 1313321 1249024 64297 4.9 1.53
find_first_and_bit: 8921 8098 823 9.2 4.56
find_next_and_bit: 9796 7176 2620 26.7 5.39
Where the statistics is significant (z-score > 3), the improvement
is ~15%.
According to the bloat-o-meter, the Image size is 10-11K less:
x86_64/defconfig:
add/remove: 32/14 grow/shrink: 61/782 up/down: 6344/-16521 (-10177)
arm64/defconfig:
add/remove: 3/2 grow/shrink: 50/714 up/down: 608/-11556 (-10948)
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Yury Norov <yury.norov@gmail.com>
This commit is contained in:
Yury Norov
parent
14a99e130f
commit
e79864f316
|
|
@ -8,9 +8,12 @@
|
|||
|
||||
#include <linux/bitops.h>
|
||||
|
||||
extern unsigned long _find_next_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long nbits,
|
||||
unsigned long start, unsigned long invert, unsigned long le);
|
||||
unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits,
|
||||
unsigned long start);
|
||||
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
|
||||
unsigned long nbits, unsigned long start);
|
||||
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
|
||||
unsigned long start);
|
||||
extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);
|
||||
extern unsigned long _find_first_and_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long size);
|
||||
|
|
@ -19,6 +22,10 @@ extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long siz
|
|||
|
||||
#ifdef __BIG_ENDIAN
|
||||
unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size);
|
||||
unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned
|
||||
long size, unsigned long offset);
|
||||
unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
|
||||
long size, unsigned long offset);
|
||||
#endif
|
||||
|
||||
#ifndef find_next_bit
|
||||
|
|
@ -45,7 +52,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
|
|||
return val ? __ffs(val) : size;
|
||||
}
|
||||
|
||||
return _find_next_bit(addr, NULL, size, offset, 0UL, 0);
|
||||
return _find_next_bit(addr, size, offset);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -75,7 +82,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
|
|||
return val ? __ffs(val) : size;
|
||||
}
|
||||
|
||||
return _find_next_bit(addr1, addr2, size, offset, 0UL, 0);
|
||||
return _find_next_and_bit(addr1, addr2, size, offset);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -103,7 +110,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
|
|||
return val == ~0UL ? size : ffz(val);
|
||||
}
|
||||
|
||||
return _find_next_bit(addr, NULL, size, offset, ~0UL, 0);
|
||||
return _find_next_zero_bit(addr, size, offset);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -251,7 +258,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned
|
|||
return val == ~0UL ? size : ffz(val);
|
||||
}
|
||||
|
||||
return _find_next_bit(addr, NULL, size, offset, ~0UL, 1);
|
||||
return _find_next_zero_bit_le(addr, size, offset);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -284,7 +291,7 @@ unsigned long find_next_bit_le(const void *addr, unsigned
|
|||
return val ? __ffs(val) : size;
|
||||
}
|
||||
|
||||
return _find_next_bit(addr, NULL, size, offset, 0UL, 1);
|
||||
return _find_next_bit_le(addr, size, offset);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
119
lib/find_bit.c
119
lib/find_bit.c
|
|
@ -40,57 +40,33 @@
|
|||
sz; \
|
||||
})
|
||||
|
||||
#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \
|
||||
!defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \
|
||||
!defined(find_next_and_bit)
|
||||
/*
|
||||
* This is a common helper function for find_next_bit, find_next_zero_bit, and
|
||||
* find_next_and_bit. The differences are:
|
||||
* - The "invert" argument, which is XORed with each fetched word before
|
||||
* searching it for one bits.
|
||||
* - The optional "addr2", which is anded with "addr1" if present.
|
||||
* Common helper for find_next_bit() function family
|
||||
* @FETCH: The expression that fetches and pre-processes each word of bitmap(s)
|
||||
* @MUNGE: The expression that post-processes a word containing found bit (may be empty)
|
||||
* @size: The bitmap size in bits
|
||||
* @start: The bitnumber to start searching at
|
||||
*/
|
||||
unsigned long _find_next_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long nbits,
|
||||
unsigned long start, unsigned long invert, unsigned long le)
|
||||
{
|
||||
unsigned long tmp, mask;
|
||||
|
||||
if (unlikely(start >= nbits))
|
||||
return nbits;
|
||||
|
||||
tmp = addr1[start / BITS_PER_LONG];
|
||||
if (addr2)
|
||||
tmp &= addr2[start / BITS_PER_LONG];
|
||||
tmp ^= invert;
|
||||
|
||||
/* Handle 1st word. */
|
||||
mask = BITMAP_FIRST_WORD_MASK(start);
|
||||
if (le)
|
||||
mask = swab(mask);
|
||||
|
||||
tmp &= mask;
|
||||
|
||||
start = round_down(start, BITS_PER_LONG);
|
||||
|
||||
while (!tmp) {
|
||||
start += BITS_PER_LONG;
|
||||
if (start >= nbits)
|
||||
return nbits;
|
||||
|
||||
tmp = addr1[start / BITS_PER_LONG];
|
||||
if (addr2)
|
||||
tmp &= addr2[start / BITS_PER_LONG];
|
||||
tmp ^= invert;
|
||||
}
|
||||
|
||||
if (le)
|
||||
tmp = swab(tmp);
|
||||
|
||||
return min(start + __ffs(tmp), nbits);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_bit);
|
||||
#endif
|
||||
#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \
|
||||
({ \
|
||||
unsigned long mask, idx, tmp, sz = (size), __start = (start); \
|
||||
\
|
||||
if (unlikely(__start >= sz)) \
|
||||
goto out; \
|
||||
\
|
||||
mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \
|
||||
idx = __start / BITS_PER_LONG; \
|
||||
\
|
||||
for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \
|
||||
if ((idx + 1) * BITS_PER_LONG >= sz) \
|
||||
goto out; \
|
||||
idx++; \
|
||||
} \
|
||||
\
|
||||
sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \
|
||||
out: \
|
||||
sz; \
|
||||
})
|
||||
|
||||
#ifndef find_first_bit
|
||||
/*
|
||||
|
|
@ -127,6 +103,32 @@ unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size
|
|||
EXPORT_SYMBOL(_find_first_zero_bit);
|
||||
#endif
|
||||
|
||||
#ifndef find_next_bit
|
||||
unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start)
|
||||
{
|
||||
return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_bit);
|
||||
#endif
|
||||
|
||||
#ifndef find_next_and_bit
|
||||
unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2,
|
||||
unsigned long nbits, unsigned long start)
|
||||
{
|
||||
return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_and_bit);
|
||||
#endif
|
||||
|
||||
#ifndef find_next_zero_bit
|
||||
unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
|
||||
unsigned long start)
|
||||
{
|
||||
return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_zero_bit);
|
||||
#endif
|
||||
|
||||
#ifndef find_last_bit
|
||||
unsigned long _find_last_bit(const unsigned long *addr, unsigned long size)
|
||||
{
|
||||
|
|
@ -175,4 +177,23 @@ EXPORT_SYMBOL(_find_first_zero_bit_le);
|
|||
|
||||
#endif
|
||||
|
||||
#ifndef find_next_zero_bit_le
|
||||
unsigned long _find_next_zero_bit_le(const unsigned long *addr,
|
||||
unsigned long size, unsigned long offset)
|
||||
{
|
||||
return FIND_NEXT_BIT(~addr[idx], swab, size, offset);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_zero_bit_le);
|
||||
#endif
|
||||
|
||||
#ifndef find_next_bit_le
|
||||
unsigned long _find_next_bit_le(const unsigned long *addr,
|
||||
unsigned long size, unsigned long offset)
|
||||
{
|
||||
return FIND_NEXT_BIT(addr[idx], swab, size, offset);
|
||||
}
|
||||
EXPORT_SYMBOL(_find_next_bit_le);
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* __BIG_ENDIAN */
|
||||
|
|
|
|||
Loading…
Reference in New Issue