mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 17:30:02 +00:00
selftests/rseq: x86-64: use %fs segment selector for accessing rseq thread area
Rather than use rseq_get_abi() and pass its result through a register to the inline assembler, directly access the per-thread rseq area through a memory reference combining the %fs segment selector, the constant offset of the field in struct rseq, and the rseq_offset value (in a register). Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/20220124171253.22072-15-mathieu.desnoyers@efficios.com
This commit is contained in:
parent
b53823fb2e
commit
4e15bb766b
1 changed files with 30 additions and 28 deletions
|
@ -28,6 +28,8 @@
|
||||||
|
|
||||||
#ifdef __x86_64__
|
#ifdef __x86_64__
|
||||||
|
|
||||||
|
#define RSEQ_ASM_TP_SEGMENT %%fs
|
||||||
|
|
||||||
#define rseq_smp_mb() \
|
#define rseq_smp_mb() \
|
||||||
__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
|
__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
|
||||||
#define rseq_smp_rmb() rseq_barrier()
|
#define rseq_smp_rmb() rseq_barrier()
|
||||||
|
@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[cmpfail]\n\t"
|
"jnz %l[cmpfail]\n\t"
|
||||||
RSEQ_INJECT_ASM(4)
|
RSEQ_INJECT_ASM(4)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[error2]\n\t"
|
"jnz %l[error2]\n\t"
|
||||||
#endif
|
#endif
|
||||||
|
@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
[v] "m" (*v),
|
[v] "m" (*v),
|
||||||
[expect] "r" (expect),
|
[expect] "r" (expect),
|
||||||
[newv] "r" (newv)
|
[newv] "r" (newv)
|
||||||
|
@ -189,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
"movq %[v], %%rbx\n\t"
|
"movq %[v], %%rbx\n\t"
|
||||||
"cmpq %%rbx, %[expectnot]\n\t"
|
"cmpq %%rbx, %[expectnot]\n\t"
|
||||||
"je %l[cmpfail]\n\t"
|
"je %l[cmpfail]\n\t"
|
||||||
RSEQ_INJECT_ASM(4)
|
RSEQ_INJECT_ASM(4)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
"movq %[v], %%rbx\n\t"
|
"movq %[v], %%rbx\n\t"
|
||||||
"cmpq %%rbx, %[expectnot]\n\t"
|
"cmpq %%rbx, %[expectnot]\n\t"
|
||||||
"je %l[error2]\n\t"
|
"je %l[error2]\n\t"
|
||||||
|
@ -212,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* final store input */
|
/* final store input */
|
||||||
[v] "m" (*v),
|
[v] "m" (*v),
|
||||||
[expectnot] "r" (expectnot),
|
[expectnot] "r" (expectnot),
|
||||||
|
@ -255,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
#endif
|
#endif
|
||||||
/* final store */
|
/* final store */
|
||||||
"addq %[count], %[v]\n\t"
|
"addq %[count], %[v]\n\t"
|
||||||
|
@ -268,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* final store input */
|
/* final store input */
|
||||||
[v] "m" (*v),
|
[v] "m" (*v),
|
||||||
[count] "er" (count)
|
[count] "er" (count)
|
||||||
|
@ -309,11 +311,11 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
#endif
|
#endif
|
||||||
/* get p+v */
|
/* get p+v */
|
||||||
"movq %[ptr], %%rbx\n\t"
|
"movq %[ptr], %%rbx\n\t"
|
||||||
|
@ -327,7 +329,7 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* final store input */
|
/* final store input */
|
||||||
[ptr] "m" (*ptr),
|
[ptr] "m" (*ptr),
|
||||||
[off] "er" (off),
|
[off] "er" (off),
|
||||||
|
@ -364,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[cmpfail]\n\t"
|
"jnz %l[cmpfail]\n\t"
|
||||||
RSEQ_INJECT_ASM(4)
|
RSEQ_INJECT_ASM(4)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[error2]\n\t"
|
"jnz %l[error2]\n\t"
|
||||||
#endif
|
#endif
|
||||||
|
@ -385,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* try store input */
|
/* try store input */
|
||||||
[v2] "m" (*v2),
|
[v2] "m" (*v2),
|
||||||
[newv2] "r" (newv2),
|
[newv2] "r" (newv2),
|
||||||
|
@ -444,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||||
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
|
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
|
||||||
#endif
|
#endif
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[cmpfail]\n\t"
|
"jnz %l[cmpfail]\n\t"
|
||||||
|
@ -454,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||||
"jnz %l[cmpfail]\n\t"
|
"jnz %l[cmpfail]\n\t"
|
||||||
RSEQ_INJECT_ASM(5)
|
RSEQ_INJECT_ASM(5)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz %l[error2]\n\t"
|
"jnz %l[error2]\n\t"
|
||||||
"cmpq %[v2], %[expect2]\n\t"
|
"cmpq %[v2], %[expect2]\n\t"
|
||||||
|
@ -467,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||||
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
RSEQ_ASM_DEFINE_ABORT(4, "", abort)
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* cmp2 input */
|
/* cmp2 input */
|
||||||
[v2] "m" (*v2),
|
[v2] "m" (*v2),
|
||||||
[expect2] "r" (expect2),
|
[expect2] "r" (expect2),
|
||||||
|
@ -524,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||||
"movq %[dst], %[rseq_scratch1]\n\t"
|
"movq %[dst], %[rseq_scratch1]\n\t"
|
||||||
"movq %[len], %[rseq_scratch2]\n\t"
|
"movq %[len], %[rseq_scratch2]\n\t"
|
||||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
|
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
|
||||||
RSEQ_INJECT_ASM(3)
|
RSEQ_INJECT_ASM(3)
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz 5f\n\t"
|
"jnz 5f\n\t"
|
||||||
RSEQ_INJECT_ASM(4)
|
RSEQ_INJECT_ASM(4)
|
||||||
#ifdef RSEQ_COMPARE_TWICE
|
#ifdef RSEQ_COMPARE_TWICE
|
||||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
|
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
|
||||||
"cmpq %[v], %[expect]\n\t"
|
"cmpq %[v], %[expect]\n\t"
|
||||||
"jnz 7f\n\t"
|
"jnz 7f\n\t"
|
||||||
#endif
|
#endif
|
||||||
|
@ -579,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||||
#endif
|
#endif
|
||||||
: /* gcc asm goto does not allow outputs */
|
: /* gcc asm goto does not allow outputs */
|
||||||
: [cpu_id] "r" (cpu),
|
: [cpu_id] "r" (cpu),
|
||||||
[rseq_abi] "r" (rseq_get_abi()),
|
[rseq_offset] "r" ((long)rseq_offset),
|
||||||
/* final store input */
|
/* final store input */
|
||||||
[v] "m" (*v),
|
[v] "m" (*v),
|
||||||
[expect] "r" (expect),
|
[expect] "r" (expect),
|
||||||
|
|
Loading…
Reference in a new issue