mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-03 23:58:05 +00:00
e9a688bcb1
Until the very recent commits, many bounded random integers were calculated using `get_random_u32() % max_plus_one`, which not only incurs the price of a division -- indicating performance mostly was not a real issue -- but also does not result in a uniformly distributed output if max_plus_one is not a power of two. Recent commits moved to using `prandom_u32_max(max_plus_one)`, which replaces the division with a faster multiplication, but still does not solve the issue with non-uniform output. For some users, maybe this isn't a problem, and for others, maybe it is, but for the majority of users, probably the question has never been posed and analyzed, and nobody thought much about it, probably assuming random is random is random. In other words, the unthinking expectation of most users is likely that the resultant numbers are uniform. So we implement here an efficient way of generating uniform bounded random integers. Through use of compile-time evaluation, and avoiding divisions as much as possible, this commit introduces no measurable overhead. At least for hot-path uses tested, any potential difference was lost in the noise. On both clang and gcc, code generation is pretty small. The new function, get_random_u32_below(), lives in random.h, rather than prandom.h, and has a "get_random_xxx" function name, because it is suitable for all uses, including cryptography. In order to be efficient, we implement a kernel-specific variant of Daniel Lemire's algorithm from "Fast Random Integer Generation in an Interval", linked below. The kernel's variant takes advantage of constant folding to avoid divisions entirely in the vast majority of cases, works on both 32-bit and 64-bit architectures, and requests a minimal amount of bytes from the RNG. Link: https://arxiv.org/pdf/1805.10941.pdf Cc: stable@vger.kernel.org # to ease future backports that use this api Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
180 lines
5.3 KiB
C
180 lines
5.3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
#ifndef _LINUX_RANDOM_H
|
|
#define _LINUX_RANDOM_H
|
|
|
|
#include <linux/bug.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/list.h>
|
|
#include <linux/once.h>
|
|
|
|
#include <uapi/linux/random.h>
|
|
|
|
struct notifier_block;
|
|
|
|
void add_device_randomness(const void *buf, size_t len);
|
|
void __init add_bootloader_randomness(const void *buf, size_t len);
|
|
void add_input_randomness(unsigned int type, unsigned int code,
|
|
unsigned int value) __latent_entropy;
|
|
void add_interrupt_randomness(int irq) __latent_entropy;
|
|
void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
|
|
|
|
#if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
|
|
static inline void add_latent_entropy(void)
|
|
{
|
|
add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
|
|
}
|
|
#else
|
|
static inline void add_latent_entropy(void) { }
|
|
#endif
|
|
|
|
#if IS_ENABLED(CONFIG_VMGENID)
|
|
void add_vmfork_randomness(const void *unique_vm_id, size_t len);
|
|
int register_random_vmfork_notifier(struct notifier_block *nb);
|
|
int unregister_random_vmfork_notifier(struct notifier_block *nb);
|
|
#else
|
|
static inline int register_random_vmfork_notifier(struct notifier_block *nb) { return 0; }
|
|
static inline int unregister_random_vmfork_notifier(struct notifier_block *nb) { return 0; }
|
|
#endif
|
|
|
|
void get_random_bytes(void *buf, size_t len);
|
|
u8 get_random_u8(void);
|
|
u16 get_random_u16(void);
|
|
u32 get_random_u32(void);
|
|
u64 get_random_u64(void);
|
|
static inline unsigned long get_random_long(void)
|
|
{
|
|
#if BITS_PER_LONG == 64
|
|
return get_random_u64();
|
|
#else
|
|
return get_random_u32();
|
|
#endif
|
|
}
|
|
|
|
u32 __get_random_u32_below(u32 ceil);
|
|
|
|
/*
|
|
* Returns a random integer in the interval [0, ceil), with uniform
|
|
* distribution, suitable for all uses. Fastest when ceil is a constant, but
|
|
* still fast for variable ceil as well.
|
|
*/
|
|
static inline u32 get_random_u32_below(u32 ceil)
|
|
{
|
|
if (!__builtin_constant_p(ceil))
|
|
return __get_random_u32_below(ceil);
|
|
|
|
/*
|
|
* For the fast path, below, all operations on ceil are precomputed by
|
|
* the compiler, so this incurs no overhead for checking pow2, doing
|
|
* divisions, or branching based on integer size. The resultant
|
|
* algorithm does traditional reciprocal multiplication (typically
|
|
* optimized by the compiler into shifts and adds), rejecting samples
|
|
* whose lower half would indicate a range indivisible by ceil.
|
|
*/
|
|
BUILD_BUG_ON_MSG(!ceil, "get_random_u32_below() must take ceil > 0");
|
|
if (ceil <= 1)
|
|
return 0;
|
|
for (;;) {
|
|
if (ceil <= 1U << 8) {
|
|
u32 mult = ceil * get_random_u8();
|
|
if (likely(is_power_of_2(ceil) || (u8)mult >= (1U << 8) % ceil))
|
|
return mult >> 8;
|
|
} else if (ceil <= 1U << 16) {
|
|
u32 mult = ceil * get_random_u16();
|
|
if (likely(is_power_of_2(ceil) || (u16)mult >= (1U << 16) % ceil))
|
|
return mult >> 16;
|
|
} else {
|
|
u64 mult = (u64)ceil * get_random_u32();
|
|
if (likely(is_power_of_2(ceil) || (u32)mult >= -ceil % ceil))
|
|
return mult >> 32;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* On 64-bit architectures, protect against non-terminated C string overflows
|
|
* by zeroing out the first byte of the canary; this leaves 56 bits of entropy.
|
|
*/
|
|
#ifdef CONFIG_64BIT
|
|
# ifdef __LITTLE_ENDIAN
|
|
# define CANARY_MASK 0xffffffffffffff00UL
|
|
# else /* big endian, 64 bits: */
|
|
# define CANARY_MASK 0x00ffffffffffffffUL
|
|
# endif
|
|
#else /* 32 bits: */
|
|
# define CANARY_MASK 0xffffffffUL
|
|
#endif
|
|
|
|
static inline unsigned long get_random_canary(void)
|
|
{
|
|
return get_random_long() & CANARY_MASK;
|
|
}
|
|
|
|
void __init random_init_early(const char *command_line);
|
|
void __init random_init(void);
|
|
bool rng_is_initialized(void);
|
|
int wait_for_random_bytes(void);
|
|
|
|
/* Calls wait_for_random_bytes() and then calls get_random_bytes(buf, nbytes).
|
|
* Returns the result of the call to wait_for_random_bytes. */
|
|
static inline int get_random_bytes_wait(void *buf, size_t nbytes)
|
|
{
|
|
int ret = wait_for_random_bytes();
|
|
get_random_bytes(buf, nbytes);
|
|
return ret;
|
|
}
|
|
|
|
#define declare_get_random_var_wait(name, ret_type) \
|
|
static inline int get_random_ ## name ## _wait(ret_type *out) { \
|
|
int ret = wait_for_random_bytes(); \
|
|
if (unlikely(ret)) \
|
|
return ret; \
|
|
*out = get_random_ ## name(); \
|
|
return 0; \
|
|
}
|
|
declare_get_random_var_wait(u8, u8)
|
|
declare_get_random_var_wait(u16, u16)
|
|
declare_get_random_var_wait(u32, u32)
|
|
declare_get_random_var_wait(u64, u32)
|
|
declare_get_random_var_wait(long, unsigned long)
|
|
#undef declare_get_random_var
|
|
|
|
/*
|
|
* This is designed to be standalone for just prandom
|
|
* users, but for now we include it from <linux/random.h>
|
|
* for legacy reasons.
|
|
*/
|
|
#include <linux/prandom.h>
|
|
|
|
#include <asm/archrandom.h>
|
|
|
|
/*
|
|
* Called from the boot CPU during startup; not valid to call once
|
|
* secondary CPUs are up and preemption is possible.
|
|
*/
|
|
#ifndef arch_get_random_seed_longs_early
|
|
static inline size_t __init arch_get_random_seed_longs_early(unsigned long *v, size_t max_longs)
|
|
{
|
|
WARN_ON(system_state != SYSTEM_BOOTING);
|
|
return arch_get_random_seed_longs(v, max_longs);
|
|
}
|
|
#endif
|
|
|
|
#ifndef arch_get_random_longs_early
|
|
static inline bool __init arch_get_random_longs_early(unsigned long *v, size_t max_longs)
|
|
{
|
|
WARN_ON(system_state != SYSTEM_BOOTING);
|
|
return arch_get_random_longs(v, max_longs);
|
|
}
|
|
#endif
|
|
|
|
#ifdef CONFIG_SMP
|
|
int random_prepare_cpu(unsigned int cpu);
|
|
int random_online_cpu(unsigned int cpu);
|
|
#endif
|
|
|
|
#ifndef MODULE
|
|
extern const struct file_operations random_fops, urandom_fops;
|
|
#endif
|
|
|
|
#endif /* _LINUX_RANDOM_H */
|