mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

powerpc64/bpf: Store temp registers' bpf to ppc mapping 

In bpf_jit_build_body(), the mapping of TMP_REG_1 and TMP_REG_2's bpf
register to ppc register is evalulated at every use despite not
changing. Instead, determine the ppc register once and store the result.

Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
[Rebased, converted additional usage sites]
Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/0944e2f0fa6dd254ea401f1c946fb6c9a5294278.1644834730.git.naveen.n.rao@linux.vnet.ibm.com
This commit is contained in:
Jordan Niethe 2022-02-14 16:11:50 +05:30 committed by Michael Ellerman
parent 036d559c0b
commit 3a3fc9bf10
1 changed files with 86 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
arch/powerpc/net/bpf_jit_comp64.c
View File

@ -357,6 +357,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
u32 dst_reg = b2p[insn[i].dst_reg]; u32 dst_reg = b2p[insn[i].dst_reg];
u32 src_reg = b2p[insn[i].src_reg]; u32 src_reg = b2p[insn[i].src_reg];
u32 size = BPF_SIZE(code); u32 size = BPF_SIZE(code);
u32 tmp1_reg = b2p[TMP_REG_1];
u32 tmp2_reg = b2p[TMP_REG_2];
s16 off = insn[i].off; s16 off = insn[i].off;
s32 imm = insn[i].imm; s32 imm = insn[i].imm;
bool func_addr_fixed; bool func_addr_fixed;
@ -407,8 +409,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
} else if (imm >= -32768 && imm < 32768) { } else if (imm >= -32768 && imm < 32768) {
EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm))); EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
} else { } else {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_ADD(dst_reg, dst_reg, tmp1_reg));
} }
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
@ -418,8 +420,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
} else if (imm > -32768 && imm <= 32768) { } else if (imm > -32768 && imm <= 32768) {
EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm))); EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(-imm)));
} else { } else {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
} }
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
@ -434,32 +436,28 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
if (imm >= -32768 && imm < 32768) if (imm >= -32768 && imm < 32768)
EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm))); EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
else { else {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
if (BPF_CLASS(code) == BPF_ALU) if (BPF_CLASS(code) == BPF_ALU)
EMIT(PPC_RAW_MULW(dst_reg, dst_reg, EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
else else
EMIT(PPC_RAW_MULD(dst_reg, dst_reg, EMIT(PPC_RAW_MULD(dst_reg, dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
} }
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
if (BPF_OP(code) == BPF_MOD) { if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg)); EMIT(PPC_RAW_DIVWU(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg, EMIT(PPC_RAW_MULW(tmp1_reg, src_reg, tmp1_reg));
b2p[TMP_REG_1])); EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else } else
EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg)); EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
if (BPF_OP(code) == BPF_MOD) { if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg)); EMIT(PPC_RAW_DIVDU(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg, EMIT(PPC_RAW_MULD(tmp1_reg, src_reg, tmp1_reg));
b2p[TMP_REG_1])); EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
} else } else
EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg)); EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
break; break;
@ -478,35 +476,23 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
} }
} }
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
switch (BPF_CLASS(code)) { switch (BPF_CLASS(code)) {
case BPF_ALU: case BPF_ALU:
if (BPF_OP(code) == BPF_MOD) { if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2], EMIT(PPC_RAW_DIVWU(tmp2_reg, dst_reg, tmp1_reg));
dst_reg, EMIT(PPC_RAW_MULW(tmp1_reg, tmp1_reg, tmp2_reg));
b2p[TMP_REG_1])); EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]));
} else } else
EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
break; break;
case BPF_ALU64: case BPF_ALU64:
if (BPF_OP(code) == BPF_MOD) { if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2], EMIT(PPC_RAW_DIVDU(tmp2_reg, dst_reg, tmp1_reg));
dst_reg, EMIT(PPC_RAW_MULD(tmp1_reg, tmp1_reg, tmp2_reg));
b2p[TMP_REG_1])); EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]));
} else } else
EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
break; break;
} }
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
@ -528,8 +514,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm))); EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
else { else {
/* Sign-extended */ /* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_AND(dst_reg, dst_reg, tmp1_reg));
} }
goto bpf_alu32_trunc; goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
@ -540,8 +526,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */ /* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp1_reg));
} else { } else {
if (IMM_L(imm)) if (IMM_L(imm))
EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm))); EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
@ -557,8 +543,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */ /* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_XOR(dst_reg, dst_reg, tmp1_reg));
} else { } else {
if (IMM_L(imm)) if (IMM_L(imm))
EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm))); EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
@ -659,11 +645,11 @@ bpf_alu32_trunc:
switch (imm) { switch (imm) {
case 16: case 16:
/* Rotate 8 bits left & mask with 0x0000ff00 */ /* Rotate 8 bits left & mask with 0x0000ff00 */
EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23)); EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 16, 23));
/* Rotate 8 bits right & insert LSB to reg */ /* Rotate 8 bits right & insert LSB to reg */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31)); EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 24, 31));
/* Move result back to dst_reg */ /* Move result back to dst_reg */
EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
break; break;
case 32: case 32:
/* /*
@ -671,28 +657,28 @@ bpf_alu32_trunc:
* 2 bytes are already in their final position * 2 bytes are already in their final position
* -- byte 2 and 4 (of bytes 1, 2, 3 and 4) * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
*/ */
EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31)); EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 0, 31));
/* Rotate 24 bits and insert byte 1 */ /* Rotate 24 bits and insert byte 1 */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7)); EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 0, 7));
/* Rotate 24 bits and insert byte 3 */ /* Rotate 24 bits and insert byte 3 */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23)); EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 16, 23));
EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
break; break;
case 64: case 64:
/* Store the value to stack and then use byte-reverse loads */ /* Store the value to stack and then use byte-reverse loads */
EMIT(PPC_RAW_STD(dst_reg, _R1, bpf_jit_stack_local(ctx))); EMIT(PPC_RAW_STD(dst_reg, _R1, bpf_jit_stack_local(ctx)));
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], _R1, bpf_jit_stack_local(ctx))); EMIT(PPC_RAW_ADDI(tmp1_reg, _R1, bpf_jit_stack_local(ctx)));
if (cpu_has_feature(CPU_FTR_ARCH_206)) { if (cpu_has_feature(CPU_FTR_ARCH_206)) {
EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1])); EMIT(PPC_RAW_LDBRX(dst_reg, 0, tmp1_reg));
} else { } else {
EMIT(PPC_RAW_LWBRX(dst_reg, 0, b2p[TMP_REG_1])); EMIT(PPC_RAW_LWBRX(dst_reg, 0, tmp1_reg));
if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN)) if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32)); EMIT(PPC_RAW_SLDI(dst_reg, dst_reg, 32));
EMIT(PPC_RAW_LI(b2p[TMP_REG_2], 4)); EMIT(PPC_RAW_LI(tmp2_reg, 4));
EMIT(PPC_RAW_LWBRX(b2p[TMP_REG_2], b2p[TMP_REG_2], b2p[TMP_REG_1])); EMIT(PPC_RAW_LWBRX(tmp2_reg, tmp2_reg, tmp1_reg));
if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
EMIT(PPC_RAW_SLDI(b2p[TMP_REG_2], b2p[TMP_REG_2], 32)); EMIT(PPC_RAW_SLDI(tmp2_reg, tmp2_reg, 32));
EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_2])); EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp2_reg));
} }
break; break;
} }
@ -731,7 +717,7 @@ emit_clear:
break; break;
case STF_BARRIER_SYNC_ORI: case STF_BARRIER_SYNC_ORI:
EMIT(PPC_RAW_SYNC()); EMIT(PPC_RAW_SYNC());
EMIT(PPC_RAW_LD(b2p[TMP_REG_1], _R13, 0)); EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0));
EMIT(PPC_RAW_ORI(_R31, _R31, 0)); EMIT(PPC_RAW_ORI(_R31, _R31, 0));
break; break;
case STF_BARRIER_FALLBACK: case STF_BARRIER_FALLBACK:
@ -751,36 +737,36 @@ emit_clear:
case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) { if (BPF_CLASS(code) == BPF_ST) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm)); EMIT(PPC_RAW_LI(tmp1_reg, imm));
src_reg = b2p[TMP_REG_1]; src_reg = tmp1_reg;
} }
EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
break; break;
case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) { if (BPF_CLASS(code) == BPF_ST) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm)); EMIT(PPC_RAW_LI(tmp1_reg, imm));
src_reg = b2p[TMP_REG_1]; src_reg = tmp1_reg;
} }
EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
break; break;
case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) { if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
src_reg = b2p[TMP_REG_1]; src_reg = tmp1_reg;
} }
EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
break; break;
case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) { if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
src_reg = b2p[TMP_REG_1]; src_reg = tmp1_reg;
} }
if (off % 4) { if (off % 4) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_2], off)); EMIT(PPC_RAW_LI(tmp2_reg, off));
EMIT(PPC_RAW_STDX(src_reg, dst_reg, b2p[TMP_REG_2])); EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
} else { } else {
EMIT(PPC_RAW_STD(src_reg, dst_reg, off)); EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
} }
@ -800,14 +786,14 @@ emit_clear:
/* *(u32 *)(dst + off) += src */ /* *(u32 *)(dst + off) += src */
/* Get EA into TMP_REG_1 */ /* Get EA into TMP_REG_1 */
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); EMIT(PPC_RAW_ADDI(tmp1_reg, dst_reg, off));
tmp_idx = ctx->idx * 4; tmp_idx = ctx->idx * 4;
/* load value from memory into TMP_REG_2 */ /* load value from memory into TMP_REG_2 */
EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0)); EMIT(PPC_RAW_LWARX(tmp2_reg, 0, tmp1_reg, 0));
/* add value from src_reg into this */ /* add value from src_reg into this */
EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg)); EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
/* store result back */ /* store result back */
EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1])); EMIT(PPC_RAW_STWCX(tmp2_reg, 0, tmp1_reg));
/* we're done if this succeeded */ /* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, tmp_idx); PPC_BCC_SHORT(COND_NE, tmp_idx);
break; break;
@ -820,11 +806,11 @@ emit_clear:
} }
/* *(u64 *)(dst + off) += src */ /* *(u64 *)(dst + off) += src */
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); EMIT(PPC_RAW_ADDI(tmp1_reg, dst_reg, off));
tmp_idx = ctx->idx * 4; tmp_idx = ctx->idx * 4;
EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0)); EMIT(PPC_RAW_LDARX(tmp2_reg, 0, tmp1_reg, 0));
EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg)); EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1])); EMIT(PPC_RAW_STDCX(tmp2_reg, 0, tmp1_reg));
PPC_BCC_SHORT(COND_NE, tmp_idx); PPC_BCC_SHORT(COND_NE, tmp_idx);
break; break;
@ -850,12 +836,12 @@ emit_clear:
* set dst_reg=0 and move on. * set dst_reg=0 and move on.
*/ */
if (BPF_MODE(code) == BPF_PROBE_MEM) { if (BPF_MODE(code) == BPF_PROBE_MEM) {
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], src_reg, off)); EMIT(PPC_RAW_ADDI(tmp1_reg, src_reg, off));
if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) if (IS_ENABLED(CONFIG_PPC_BOOK3E_64))
PPC_LI64(b2p[TMP_REG_2], 0x8000000000000000ul); PPC_LI64(tmp2_reg, 0x8000000000000000ul);
else /* BOOK3S_64 */ else /* BOOK3S_64 */
PPC_LI64(b2p[TMP_REG_2], PAGE_OFFSET); PPC_LI64(tmp2_reg, PAGE_OFFSET);
EMIT(PPC_RAW_CMPLD(b2p[TMP_REG_1], b2p[TMP_REG_2])); EMIT(PPC_RAW_CMPLD(tmp1_reg, tmp2_reg));
PPC_BCC_SHORT(COND_GT, (ctx->idx + 3) * 4); PPC_BCC_SHORT(COND_GT, (ctx->idx + 3) * 4);
EMIT(PPC_RAW_LI(dst_reg, 0)); EMIT(PPC_RAW_LI(dst_reg, 0));
/* /*
@ -880,8 +866,8 @@ emit_clear:
break; break;
case BPF_DW: case BPF_DW:
if (off % 4) { if (off % 4) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_1], off)); EMIT(PPC_RAW_LI(tmp1_reg, off));
EMIT(PPC_RAW_LDX(dst_reg, src_reg, b2p[TMP_REG_1])); EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
} else { } else {
EMIT(PPC_RAW_LD(dst_reg, src_reg, off)); EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
} }
@ -925,7 +911,7 @@ emit_clear:
* we'll just fall through to the epilogue. * we'll just fall through to the epilogue.
*/ */
if (i != flen - 1) { if (i != flen - 1) {
ret = bpf_jit_emit_exit_insn(image, ctx, b2p[TMP_REG_1], exit_addr); ret = bpf_jit_emit_exit_insn(image, ctx, tmp1_reg, exit_addr);
if (ret) if (ret)
return ret; return ret;
} }
@ -1058,14 +1044,10 @@ cond_branch:
case BPF_JMP | BPF_JSET | BPF_X: case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X: case BPF_JMP32 | BPF_JSET | BPF_X:
if (BPF_CLASS(code) == BPF_JMP) { if (BPF_CLASS(code) == BPF_JMP) {
EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg, EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, src_reg));
src_reg));
} else { } else {
int tmp_reg = b2p[TMP_REG_1]; EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 0, 0, 31));
EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
31));
} }
break; break;
case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP | BPF_JNE | BPF_K:
@ -1094,14 +1076,12 @@ cond_branch:
EMIT(PPC_RAW_CMPLDI(dst_reg, imm)); EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
} else { } else {
/* sign-extending load */ /* sign-extending load */
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
/* ... but unsigned comparison */ /* ... but unsigned comparison */
if (is_jmp32) if (is_jmp32)
EMIT(PPC_RAW_CMPLW(dst_reg, EMIT(PPC_RAW_CMPLW(dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
else else
EMIT(PPC_RAW_CMPLD(dst_reg, EMIT(PPC_RAW_CMPLD(dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
} }
break; break;
} }
@ -1126,13 +1106,11 @@ cond_branch:
else else
EMIT(PPC_RAW_CMPDI(dst_reg, imm)); EMIT(PPC_RAW_CMPDI(dst_reg, imm));
} else { } else {
PPC_LI32(b2p[TMP_REG_1], imm); PPC_LI32(tmp1_reg, imm);
if (is_jmp32) if (is_jmp32)
EMIT(PPC_RAW_CMPW(dst_reg, EMIT(PPC_RAW_CMPW(dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
else else
EMIT(PPC_RAW_CMPD(dst_reg, EMIT(PPC_RAW_CMPD(dst_reg, tmp1_reg));
b2p[TMP_REG_1]));
} }
break; break;
} }
@ -1141,19 +1119,16 @@ cond_branch:
/* andi does not sign-extend the immediate */ /* andi does not sign-extend the immediate */
if (imm >= 0 && imm < 32768) if (imm >= 0 && imm < 32768)
/* PPC_ANDI is _only/always_ dot-form */ /* PPC_ANDI is _only/always_ dot-form */
EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm)); EMIT(PPC_RAW_ANDI(tmp1_reg, dst_reg, imm));
else { else {
int tmp_reg = b2p[TMP_REG_1]; PPC_LI32(tmp1_reg, imm);
PPC_LI32(tmp_reg, imm);
if (BPF_CLASS(code) == BPF_JMP) { if (BPF_CLASS(code) == BPF_JMP) {
EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg, EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg,
tmp_reg)); tmp1_reg));
} else { } else {
EMIT(PPC_RAW_AND(tmp_reg, dst_reg, EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, tmp1_reg));
tmp_reg)); EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg,
EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0, 31));
0, 0, 31));
} }
} }
break; break;