mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge patch series "riscv: mm: Extend mappable memory up to hint address" 

Charlie Jenkins <charlie@rivosinc.com> says:

On riscv, mmap currently returns an address from the largest address
space that can fit entirely inside of the hint address. This makes it
such that the hint address is almost never returned. This patch raises
the mappable area up to and including the hint address. This allows mmap
to often return the hint address, which allows a performance improvement
over searching for a valid address as well as making the behavior more
similar to other architectures.

Note that a previous patch introduced stronger semantics compared to
other architectures for riscv mmap. On riscv, mmap will not use bits in
the upper bits of the virtual address depending on the hint address. On
other architectures, a random address is returned in the address space
requested. On all architectures the hint address will be returned if it
is available. This allows riscv applications to configure how many bits
in the virtual address should be left empty. This has the two benefits
of being able to request address spaces that are smaller than the
default and doesn't require the application to know the page table
layout of riscv.

* b4-shazam-merge:
  docs: riscv: Define behavior of mmap
  selftests: riscv: Generalize mm selftests
  riscv: mm: Use hint address in mmap if available

Link: https://lore.kernel.org/r/20240130-use_mmap_hint_address-v3-0-8a655cfa8bcb@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
This commit is contained in:
Palmer Dabbelt 2024-03-15 10:17:34 -07:00
parent 099dbac6e9 371a3c2055
commit 07f2c040fa
No known key found for this signature in database
GPG Key ID: 2E1319F35FBB1889
5 changed files with 83 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Documentation/arch/riscv/vm-layout.rst
View File

@ -144,14 +144,8 @@ passing 0 into the hint address parameter of mmap. On CPUs with an address space
smaller than sv48, the CPU maximum supported address space will be the default.
Software can "opt-in" to receiving VAs from another VA space by providing
a hint address to mmap. A hint address passed to mmap will cause the largest
address space that fits entirely into the hint to be used, unless there is no
space left in the address space. If there is no space available in the requested
address space, an address in the next smallest available address space will be
returned.
For example, in order to obtain 48-bit VA space, a hint address greater than
:code:`1 << 47` must be provided. Note that this is 47 due to sv48 userspace
ending at :code:`1 << 47` and the addresses beyond this are reserved for the
kernel. Similarly, to obtain 57-bit VA space addresses, a hint address greater
than or equal to :code:`1 << 56` must be provided.
a hint address to mmap. When a hint address is passed to mmap, the returned
address will never use more bits than the hint address. For example, if a hint
address of `1 << 40` is passed to mmap, a valid returned address will never use
bits 41 through 63. If no mappable addresses are available in that range, mmap
will return `MAP_FAILED`.

27
arch/riscv/include/asm/processor.h
View File

@ -14,22 +14,16 @@
#include <asm/ptrace.h>
#ifdef CONFIG_64BIT
#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1))
#define STACK_TOP_MAX TASK_SIZE
#define arch_get_mmap_end(addr, len, flags) \
({ \
unsigned long mmap_end; \
typeof(addr) _addr = (addr); \
if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
if ((_addr) == 0 || \
(IS_ENABLED(CONFIG_COMPAT) && is_compat_task()) || \
((_addr + len) > BIT(VA_BITS - 1))) \
mmap_end = STACK_TOP_MAX; \
else if ((_addr) >= VA_USER_SV57) \
mmap_end = STACK_TOP_MAX; \
else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
mmap_end = VA_USER_SV48; \
else \
mmap_end = VA_USER_SV39; \
mmap_end = (_addr + len); \
mmap_end; \
})
@ -39,17 +33,18 @@
typeof(addr) _addr = (addr); \
typeof(base) _base = (base); \
unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \
if ((_addr) == 0 || (IS_ENABLED(CONFIG_COMPAT) && is_compat_task())) \
if ((_addr) == 0 || \
(IS_ENABLED(CONFIG_COMPAT) && is_compat_task()) || \
((_addr + len) > BIT(VA_BITS - 1))) \
mmap_base = (_base); \
else if (((_addr) >= VA_USER_SV57) && (VA_BITS >= VA_BITS_SV57)) \
mmap_base = VA_USER_SV57 - rnd_gap; \
else if ((((_addr) >= VA_USER_SV48)) && (VA_BITS >= VA_BITS_SV48)) \
mmap_base = VA_USER_SV48 - rnd_gap; \
else \
mmap_base = VA_USER_SV39 - rnd_gap; \
mmap_base = (_addr + len) - rnd_gap; \
mmap_base; \
})
#ifdef CONFIG_64BIT
#define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1))
#define STACK_TOP_MAX TASK_SIZE_64
#else
#define DEFAULT_MAP_WINDOW TASK_SIZE
#define STACK_TOP_MAX TASK_SIZE

23
tools/testing/selftests/riscv/mm/mmap_bottomup.c
View File

@ -6,30 +6,9 @@
TEST(infinite_rlimit)
{
// Only works on 64 bit
#if __riscv_xlen == 64
struct addresses mmap_addresses;
EXPECT_EQ(BOTTOM_UP, memory_layout());
do_mmaps(&mmap_addresses);
EXPECT_NE(MAP_FAILED, mmap_addresses.no_hint);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_37_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_38_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_46_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_47_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_55_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_56_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.no_hint);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_37_addr);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_38_addr);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_46_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_47_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_55_addr);
EXPECT_GT(1UL << 56, (unsigned long)mmap_addresses.on_56_addr);
#endif
TEST_MMAPS;
}
TEST_HARNESS_MAIN

23
tools/testing/selftests/riscv/mm/mmap_default.c
View File

@ -6,30 +6,9 @@
TEST(default_rlimit)
{
// Only works on 64 bit
#if __riscv_xlen == 64
struct addresses mmap_addresses;
EXPECT_EQ(TOP_DOWN, memory_layout());
do_mmaps(&mmap_addresses);
EXPECT_NE(MAP_FAILED, mmap_addresses.no_hint);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_37_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_38_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_46_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_47_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_55_addr);
EXPECT_NE(MAP_FAILED, mmap_addresses.on_56_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.no_hint);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_37_addr);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_38_addr);
EXPECT_GT(1UL << 38, (unsigned long)mmap_addresses.on_46_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_47_addr);
EXPECT_GT(1UL << 47, (unsigned long)mmap_addresses.on_55_addr);
EXPECT_GT(1UL << 56, (unsigned long)mmap_addresses.on_56_addr);
#endif
TEST_MMAPS;
}
TEST_HARNESS_MAIN

107
tools/testing/selftests/riscv/mm/mmap_test.h
View File

@ -4,63 +4,86 @@
#include <sys/mman.h>
#include <sys/resource.h>
#include <stddef.h>
#include <strings.h>
#include "../../kselftest_harness.h"
#define TOP_DOWN 0
#define BOTTOM_UP 1
struct addresses {
int *no_hint;
int *on_37_addr;
int *on_38_addr;
int *on_46_addr;
int *on_47_addr;
int *on_55_addr;
int *on_56_addr;
#if __riscv_xlen == 64
uint64_t random_addresses[] = {
0x19764f0d73b3a9f0, 0x016049584cecef59, 0x3580bdd3562f4acd,
0x1164219f20b17da0, 0x07d97fcb40ff2373, 0x76ec528921272ee7,
0x4dd48c38a3de3f70, 0x2e11415055f6997d, 0x14b43334ac476c02,
0x375a60795aff19f6, 0x47f3051725b8ee1a, 0x4e697cf240494a9f,
0x456b59b5c2f9e9d1, 0x101724379d63cb96, 0x7fe9ad31619528c1,
0x2f417247c495c2ea, 0x329a5a5b82943a5e, 0x06d7a9d6adcd3827,
0x327b0b9ee37f62d5, 0x17c7b1851dfd9b76, 0x006ebb6456ec2cd9,
0x00836cd14146a134, 0x00e5c4dcde7126db, 0x004c29feadf75753,
0x00d8b20149ed930c, 0x00d71574c269387a, 0x0006ebe4a82acb7a,
0x0016135df51f471b, 0x00758bdb55455160, 0x00d0bdd949b13b32,
0x00ecea01e7c5f54b, 0x00e37b071b9948b1, 0x0011fdd00ff57ab3,
0x00e407294b52f5ea, 0x00567748c200ed20, 0x000d073084651046,
0x00ac896f4365463c, 0x00eb0d49a0b26216, 0x0066a2564a982a31,
0x002e0d20237784ae, 0x0000554ff8a77a76, 0x00006ce07a54c012,
0x000009570516d799, 0x00000954ca15b84d, 0x0000684f0d453379,
0x00002ae5816302b5, 0x0000042403fb54bf, 0x00004bad7392bf30,
0x00003e73bfa4b5e3, 0x00005442c29978e0, 0x00002803f11286b6,
0x000073875d745fc6, 0x00007cede9cb8240, 0x000027df84cc6a4f,
0x00006d7e0e74242a, 0x00004afd0b836e02, 0x000047d0e837cd82,
0x00003b42405efeda, 0x00001531bafa4c95, 0x00007172cae34ac4,
};
#else
uint32_t random_addresses[] = {
0x8dc302e0, 0x929ab1e0, 0xb47683ba, 0xea519c73, 0xa19f1c90, 0xc49ba213,
0x8f57c625, 0xadfe5137, 0x874d4d95, 0xaa20f09d, 0xcf21ebfc, 0xda7737f1,
0xcedf392a, 0x83026c14, 0xccedca52, 0xc6ccf826, 0xe0cd9415, 0x997472ca,
0xa21a44c1, 0xe82196f5, 0xa23fd66b, 0xc28d5590, 0xd009cdce, 0xcf0be646,
0x8fc8c7ff, 0xe2a85984, 0xa3d3236b, 0x89a0619d, 0xc03db924, 0xb5d4cc1b,
0xb96ee04c, 0xd191da48, 0xb432a000, 0xaa2bebbc, 0xa2fcb289, 0xb0cca89b,
0xb0c18d6a, 0x88f58deb, 0xa4d42d1c, 0xe4d74e86, 0x99902b09, 0x8f786d31,
0xbec5e381, 0x9a727e65, 0xa9a65040, 0xa880d789, 0x8f1b335e, 0xfc821c1e,
0x97e34be4, 0xbbef84ed, 0xf447d197, 0xfd7ceee2, 0xe632348d, 0xee4590f4,
0x958992a5, 0xd57e05d6, 0xfd240970, 0xc5b0dcff, 0xd96da2c2, 0xa7ae041d,
};
#endif
// Only works on 64 bit
#if __riscv_xlen == 64
static inline void do_mmaps(struct addresses *mmap_addresses)
#define PROT (PROT_READ | PROT_WRITE)
#define FLAGS (MAP_PRIVATE | MAP_ANONYMOUS)
/* mmap must return a value that doesn't use more bits than the hint address. */
static inline unsigned long get_max_value(unsigned long input)
{
/*
* Place all of the hint addresses on the boundaries of mmap
* sv39, sv48, sv57
* User addresses end at 1<<38, 1<<47, 1<<56 respectively
*/
void *on_37_bits = (void *)(1UL << 37);
void *on_38_bits = (void *)(1UL << 38);
void *on_46_bits = (void *)(1UL << 46);
void *on_47_bits = (void *)(1UL << 47);
void *on_55_bits = (void *)(1UL << 55);
void *on_56_bits = (void *)(1UL << 56);
unsigned long max_bit = (1UL << (((sizeof(unsigned long) * 8) - 1 -
__builtin_clzl(input))));
int prot = PROT_READ | PROT_WRITE;
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
mmap_addresses->no_hint =
mmap(NULL, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_37_addr =
mmap(on_37_bits, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_38_addr =
mmap(on_38_bits, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_46_addr =
mmap(on_46_bits, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_47_addr =
mmap(on_47_bits, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_55_addr =
mmap(on_55_bits, 5 * sizeof(int), prot, flags, 0, 0);
mmap_addresses->on_56_addr =
mmap(on_56_bits, 5 * sizeof(int), prot, flags, 0, 0);
return max_bit + (max_bit - 1);
}
#define TEST_MMAPS \
({ \
void *mmap_addr; \
for (int i = 0; i < ARRAY_SIZE(random_addresses); i++) { \
mmap_addr = mmap((void *)random_addresses[i], \
5 * sizeof(int), PROT, FLAGS, 0, 0); \
EXPECT_NE(MAP_FAILED, mmap_addr); \
EXPECT_GE((void *)get_max_value(random_addresses[i]), \
mmap_addr); \
mmap_addr = mmap((void *)random_addresses[i], \
5 * sizeof(int), PROT, FLAGS, 0, 0); \
EXPECT_NE(MAP_FAILED, mmap_addr); \
EXPECT_GE((void *)get_max_value(random_addresses[i]), \
mmap_addr); \
} \
})
#endif /* __riscv_xlen == 64 */
static inline int memory_layout(void)
{
int prot = PROT_READ | PROT_WRITE;
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
void *value1 = mmap(NULL, sizeof(int), prot, flags, 0, 0);
void *value2 = mmap(NULL, sizeof(int), prot, flags, 0, 0);
void *value1 = mmap(NULL, sizeof(int), PROT, FLAGS, 0, 0);
void *value2 = mmap(NULL, sizeof(int), PROT, FLAGS, 0, 0);
return value2 > value1;
}