2020-01-21 12:58:52 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
|
#ifndef _ASM_ARCHRANDOM_H
|
|
|
|
|
#define _ASM_ARCHRANDOM_H
|
|
|
|
|
|
2021-01-06 10:34:52 +00:00
|
|
|
#include <linux/arm-smccc.h>
|
2020-02-10 13:00:14 +00:00
|
|
|
#include <linux/bug.h>
|
|
|
|
|
#include <linux/kernel.h>
|
2022-10-28 23:18:04 +00:00
|
|
|
#include <linux/irqflags.h>
|
2020-01-21 12:58:52 +00:00
|
|
|
#include <asm/cpufeature.h>
|
|
|
|
|
|
2021-01-06 10:34:52 +00:00
|
|
|
#define ARM_SMCCC_TRNG_MIN_VERSION 0x10000UL
|
|
|
|
|
|
|
|
|
|
extern bool smccc_trng_available;
|
|
|
|
|
|
2021-01-06 10:34:50 +00:00
|
|
|
static inline bool __init smccc_probe_trng(void)
|
|
|
|
|
{
|
2021-01-06 10:34:52 +00:00
|
|
|
struct arm_smccc_res res;
|
|
|
|
|
|
|
|
|
|
arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_VERSION, &res);
|
|
|
|
|
if ((s32)res.a0 < 0)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
return res.a0 >= ARM_SMCCC_TRNG_MIN_VERSION;
|
2021-01-06 10:34:50 +00:00
|
|
|
}
|
|
|
|
|
|
2020-01-21 12:58:52 +00:00
|
|
|
static inline bool __arm64_rndr(unsigned long *v)
|
|
|
|
|
{
|
|
|
|
|
bool ok;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reads of RNDR set PSTATE.NZCV to 0b0000 on success,
|
|
|
|
|
* and set PSTATE.NZCV to 0b0100 otherwise.
|
|
|
|
|
*/
|
|
|
|
|
asm volatile(
|
|
|
|
|
__mrs_s("%0", SYS_RNDR_EL0) "\n"
|
|
|
|
|
" cset %w1, ne\n"
|
|
|
|
|
: "=r" (*v), "=r" (ok)
|
|
|
|
|
:
|
|
|
|
|
: "cc");
|
|
|
|
|
|
|
|
|
|
return ok;
|
|
|
|
|
}
|
|
|
|
|
|
2022-01-13 13:12:39 +00:00
|
|
|
static inline bool __arm64_rndrrs(unsigned long *v)
|
|
|
|
|
{
|
|
|
|
|
bool ok;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reads of RNDRRS set PSTATE.NZCV to 0b0000 on success,
|
|
|
|
|
* and set PSTATE.NZCV to 0b0100 otherwise.
|
|
|
|
|
*/
|
|
|
|
|
asm volatile(
|
|
|
|
|
__mrs_s("%0", SYS_RNDRRS_EL0) "\n"
|
|
|
|
|
" cset %w1, ne\n"
|
|
|
|
|
: "=r" (*v), "=r" (ok)
|
|
|
|
|
:
|
|
|
|
|
: "cc");
|
|
|
|
|
|
|
|
|
|
return ok;
|
|
|
|
|
}
|
|
|
|
|
|
2022-10-28 23:18:04 +00:00
|
|
|
static __always_inline bool __cpu_has_rng(void)
|
|
|
|
|
{
|
|
|
|
|
if (unlikely(!system_capabilities_finalized() && !preemptible()))
|
|
|
|
|
return this_cpu_has_cap(ARM64_HAS_RNG);
|
arm64: Avoid cpus_have_const_cap() for ARM64_HAS_RNG
In __cpu_has_rng() we use cpus_have_const_cap() to check for
ARM64_HAS_RNG, but this is not necessary and alternative_has_cap_*()
would be preferable.
For historical reasons, cpus_have_const_cap() is more complicated than
it needs to be. Before cpucaps are finalized, it will perform a bitmap
test of the system_cpucaps bitmap, and once cpucaps are finalized it
will use an alternative branch. This used to be necessary to handle some
race conditions in the window between cpucap detection and the
subsequent patching of alternatives and static branches, where different
branches could be out-of-sync with one another (or w.r.t. alternative
sequences). Now that we use alternative branches instead of static
branches, these are all patched atomically w.r.t. one another, and there
are only a handful of cases that need special care in the window between
cpucap detection and alternative patching.
Due to the above, it would be nice to remove cpus_have_const_cap(), and
migrate callers over to alternative_has_cap_*(), cpus_have_final_cap(),
or cpus_have_cap() depending on when their requirements. This will
remove redundant instructions and improve code generation, and will make
it easier to determine how each callsite will behave before, during, and
after alternative patching.
In the window between detecting the ARM64_HAS_RNG cpucap and patching
alternative branches, nothing which calls __cpu_has_rng() can run, and
hence it's not necessary to use cpus_have_const_cap().
This patch replaces the use of cpus_have_const_cap() with
alternative_has_cap_unlikely(), which will avoid generating code to test
the system_cpucaps bitmap and should be better for all subsequent calls
at runtime.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2023-10-16 10:24:46 +00:00
|
|
|
return alternative_has_cap_unlikely(ARM64_HAS_RNG);
|
2022-10-28 23:18:04 +00:00
|
|
|
}
|
|
|
|
|
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs)
|
2020-01-21 12:58:52 +00:00
|
|
|
{
|
2022-01-13 13:12:39 +00:00
|
|
|
/*
|
|
|
|
|
* Only support the generic interface after we have detected
|
|
|
|
|
* the system wide capability, avoiding complexity with the
|
|
|
|
|
* cpufeature code and with potential scheduling between CPUs
|
|
|
|
|
* with and without the feature.
|
|
|
|
|
*/
|
2022-10-28 23:18:04 +00:00
|
|
|
if (max_longs && __cpu_has_rng() && __arm64_rndr(v))
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
return 1;
|
|
|
|
|
return 0;
|
2020-01-21 12:58:52 +00:00
|
|
|
}
|
|
|
|
|
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs)
|
2020-01-21 12:58:52 +00:00
|
|
|
{
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
if (!max_longs)
|
|
|
|
|
return 0;
|
2021-01-06 10:34:52 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We prefer the SMCCC call, since its semantics (return actual
|
|
|
|
|
* hardware backed entropy) is closer to the idea behind this
|
|
|
|
|
* function here than what even the RNDRSS register provides
|
|
|
|
|
* (the output of a pseudo RNG freshly seeded by a TRNG).
|
|
|
|
|
*/
|
|
|
|
|
if (smccc_trng_available) {
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
struct arm_smccc_res res;
|
|
|
|
|
|
|
|
|
|
max_longs = min_t(size_t, 3, max_longs);
|
|
|
|
|
arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND64, max_longs * 64, &res);
|
2021-01-06 10:34:52 +00:00
|
|
|
if ((int)res.a0 >= 0) {
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
switch (max_longs) {
|
|
|
|
|
case 3:
|
|
|
|
|
*v++ = res.a1;
|
|
|
|
|
fallthrough;
|
|
|
|
|
case 2:
|
|
|
|
|
*v++ = res.a2;
|
|
|
|
|
fallthrough;
|
|
|
|
|
case 1:
|
|
|
|
|
*v++ = res.a3;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return max_longs;
|
2021-01-06 10:34:52 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-01-21 12:58:52 +00:00
|
|
|
/*
|
2022-01-13 13:12:39 +00:00
|
|
|
* RNDRRS is not backed by an entropy source but by a DRBG that is
|
|
|
|
|
* reseeded after each invocation. This is not a 100% fit but good
|
|
|
|
|
* enough to implement this API if no other entropy source exists.
|
2020-01-21 12:58:52 +00:00
|
|
|
*/
|
2022-10-28 23:18:04 +00:00
|
|
|
if (__cpu_has_rng() && __arm64_rndrrs(v))
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
return 1;
|
2020-01-21 12:58:52 +00:00
|
|
|
|
random: handle archrandom with multiple longs
The archrandom interface was originally designed for x86, which supplies
RDRAND/RDSEED for receiving random words into registers, resulting in
one function to generate an int and another to generate a long. However,
other architectures don't follow this.
On arm64, the SMCCC TRNG interface can return between one and three
longs. On s390, the CPACF TRNG interface can return arbitrary amounts,
with four longs having the same cost as one. On UML, the os_getrandom()
interface can return arbitrary amounts.
So change the api signature to take a "max_longs" parameter designating
the maximum number of longs requested, and then return the number of
longs generated.
Since callers need to check this return value and loop anyway, each arch
implementation does not bother implementing its own loop to try again to
fill the maximum number of longs. Additionally, all existing callers
pass in a constant max_longs parameter. Taken together, these two things
mean that the codegen doesn't really change much for one-word-at-a-time
platforms, while performance is greatly improved on platforms such as
s390.
Acked-by: Heiko Carstens <hca@linux.ibm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Acked-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
2022-07-17 10:35:24 +00:00
|
|
|
return 0;
|
2020-01-21 12:58:52 +00:00
|
|
|
}
|
|
|
|
|
|
2020-01-21 12:58:53 +00:00
|
|
|
static inline bool __init __early_cpu_has_rndr(void)
|
|
|
|
|
{
|
|
|
|
|
/* Open code as we run prior to the first call to cpufeature. */
|
|
|
|
|
unsigned long ftr = read_sysreg_s(SYS_ID_AA64ISAR0_EL1);
|
2022-05-03 17:02:28 +00:00
|
|
|
return (ftr >> ID_AA64ISAR0_EL1_RNDR_SHIFT) & 0xf;
|
2020-01-21 12:58:53 +00:00
|
|
|
}
|
|
|
|
|
|
2020-01-21 12:58:52 +00:00
|
|
|
#endif /* _ASM_ARCHRANDOM_H */
|