powerpc64/bpf: Add jit support for load_acquire and store_release
Subsystems (6)
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH RESEND 1/1 powerpc64/bpf: Add jit support for load_acquire and store_release
Add JIT support for the load_acquire and store_release instructions. The implementation is similar to the kernel where:
load_acquire => plain load -> lwsync
store_release => lwsync -> plain storeTo test the correctness of the implementation, following selftests were run:
[fedora@linux-kernel bpf]$ sudo ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #519/1 verifier_load_acquire/load-acquire, 8-bit:OK #519/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #519/3 verifier_load_acquire/load-acquire, 16-bit:OK #519/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #519/5 verifier_load_acquire/load-acquire, 32-bit:OK #519/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #519/7 verifier_load_acquire/load-acquire, 64-bit:OK #519/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #519/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #519/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #519/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #519/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #519/13 verifier_load_acquire/misaligned load-acquire:OK #519/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #519/15 verifier_load_acquire/load-acquire from ctx pointer:OK #519/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #519/17 verifier_load_acquire/load-acquire with invalid register R15:OK #519/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #519/19 verifier_load_acquire/load-acquire from pkt pointer:OK #519/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #519/21 verifier_load_acquire/load-acquire from sock pointer:OK #519 verifier_load_acquire:OK #556/1 verifier_store_release/store-release, 8-bit:OK #556/2 verifier_store_release/store-release, 8-bit @unpriv:OK #556/3 verifier_store_release/store-release, 16-bit:OK #556/4 verifier_store_release/store-release, 16-bit @unpriv:OK #556/5 verifier_store_release/store-release, 32-bit:OK #556/6 verifier_store_release/store-release, 32-bit @unpriv:OK #556/7 verifier_store_release/store-release, 64-bit:OK #556/8 verifier_store_release/store-release, 64-bit @unpriv:OK #556/9 verifier_store_release/store-release with uninitialized src_reg:OK #556/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #556/11 verifier_store_release/store-release with uninitialized dst_reg:OK #556/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #556/13 verifier_store_release/store-release with non-pointer dst_reg:OK #556/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #556/15 verifier_store_release/misaligned store-release:OK #556/16 verifier_store_release/misaligned store-release @unpriv:OK #556/17 verifier_store_release/store-release to ctx pointer:OK #556/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #556/19 verifier_store_release/store-release, leak pointer to stack:OK #556/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #556/21 verifier_store_release/store-release, leak pointer to map:OK #556/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #556/23 verifier_store_release/store-release with invalid register R15:OK #556/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #556/25 verifier_store_release/store-release to pkt pointer:OK #556/26 verifier_store_release/store-release to flow_keys pointer:OK #556/27 verifier_store_release/store-release to sock pointer:OK #556 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
--- arch/powerpc/include/asm/ppc-opcode.h | 1 + arch/powerpc/net/bpf_jit_comp64.c | 82 ++++++++++++++++++++ tools/testing/selftests/bpf/progs/bpf_misc.h | 3 +- 3 files changed, 85 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/ppc-opcode.h +1 -0
@@ -425,6 +425,7 @@#define PPC_RAW_SC () (0x44000002)#define PPC_RAW_SYNC () (0x7c0004ac)#define PPC_RAW_ISYNC () (0x4c00012c) +#define PPC_RAW_LWSYNC () (0x7c2004ac)/* * Define what the VSX XX1 form instructions will look like, then add
arch/powerpc/net/bpf_jit_comp64.c +82 -0
96 lines not on this page · raw message
tools/testing/selftests/bpf/progs/bpf_misc.h +2 -1
@@ -229,7 +229,8 @@#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 )) + (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 )) || \ + (defined(__TARGET_ARCH_powerpc))#define CAN_USE_LOAD_ACQ_STORE_REL #endif
Signature · 1 line
2.47.1
Puranjay Mohan <[email protected]> writes:
Somehow the cover letter for this patch was missed, adding it here:
To test the functionality of these special instructions, a tool called blitmus[0] was used to convert the following baseline litmus test[1] to bpf programs:
C MP+poonceonces
(*
* Result: Sometimes
*
* Can the counter-intuitive message-passing outcome be prevented with
* no ordering at all?
*)
{}
P0(int *buf, int *flag)
{
WRITE_ONCE(*buf, 1);
WRITE_ONCE(*flag, 1);}
P1(int *buf, int *flag)
{
int r0;
int r1;
r0 = READ_ONCE(*flag);
r1 = READ_ONCE(*buf);}
exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)
Running the generated bpf program shows that the bad outcome is possible on powerpc:
[fedora@linux-kernel blitmus]$ sudo ./mp_poonceonces Starting litmus test with configuration:
Test: MP+poonceonces
Iterations: 4100
Test MP+poonceonces Allowed
Histogram (4 states)
21548375 :>1:r0=0; 1:r1=0;
301187 :>1:r0=0; 1:r1=1;
337147 *>1:r0=1; 1:r1=0;
18813291 :>1:r0=1; 1:r1=1;
Ok
Witnesses
Positive: 337147, Negative: 40662853 Condition exists (1:r0=1 /\ 1:r1=0) is validated Observation MP+poonceonces Sometimes 337147 40662853 Time MP+poonceonces 13.48
Thu Jul 17 18:12:51 UTC
Now the second write and the first read is converted to store_release and load_acquire and it gives us the following litmus test[2]
C MP+pooncerelease+poacquireonce
(*
* Result: Never
*
* This litmus test demonstrates that smp_store_release() and
* smp_load_acquire() provide sufficient ordering for the message-passing
* pattern.
*)
{}
P0(int *buf, int *flag)
{
WRITE_ONCE(*buf, 1);
smp_store_release(flag, 1);}
P1(int *buf, int *flag)
{
int r0;
int r1;
r0 = smp_load_acquire(flag);
r1 = READ_ONCE(*buf);}
exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)
Running the generated bpf program shows that the bad outcome is *not* possible on powerpc with the implementation in this patch:
[fedora@linux-kernel blitmus]$ sudo ./mp_pooncerelease_poacquireonce Starting litmus test with configuration:
Test: MP+pooncerelease+poacquireonce
Iterations: 4100
Test MP+pooncerelease+poacquireonce Allowed
Histogram (3 states)
21036021 :>1:r0=0; 1:r1=0;
14488694 :>1:r0=0; 1:r1=1;
5475285 :>1:r0=1; 1:r1=1;
No
Witnesses
Positive: 0, Negative: 41000000
Condition exists (1:r0=1 /\ 1:r1=0) is NOT validated Observation MP+pooncerelease+poacquireonce Never 0 41000000 Time MP+pooncerelease+poacquireonce 13.74
Thu Jul 17 18:13:40 UTC
[0] https://github.com/puranjaymohan/blitmus
[1] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpoonceonces.litmus
[2] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpooncerelease%2Bpoacquireonce.litmus
Saket Kumar Bhaskar
Tested-byQuoted 114 lines · On Thu, Jul 17, 2025 at 08:56:45PM +0000, [email protected] wrote:
Puranjay Mohan <[email protected]> writes:
Somehow the cover letter for this patch was missed, adding it here:
To test the functionality of these special instructions, a tool called blitmus[0] was used to convert the following baseline litmus test[1] to bpf programs:
C MP+poonceonces
(*
* Result: Sometimes
*
* Can the counter-intuitive message-passing outcome be prevented with
* no ordering at all?
*){}
P0(int *buf, int *flag){ WRITE_ONCE(*buf, 1); WRITE_ONCE(*flag, 1);}
P1(int *buf, int *flag){ int r0; int r1; r0 = READ_ONCE(*flag); r1 = READ_ONCE(*buf);}
exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)
Running the generated bpf program shows that the bad outcome is possible on powerpc:
[fedora@linux-kernel blitmus]$ sudo ./mp_poonceonces Starting litmus test with configuration:
Test: MP+poonceonces Iterations: 4100Test MP+poonceonces Allowed
Histogram (4 states)
21548375 :>1:r0=0; 1:r1=0;
301187 :>1:r0=0; 1:r1=1;
337147 *>1:r0=1; 1:r1=0;
18813291 :>1:r0=1; 1:r1=1;
OkWitnesses
Positive: 337147, Negative: 40662853 Condition exists (1:r0=1 /\ 1:r1=0) is validated Observation MP+poonceonces Sometimes 337147 40662853 Time MP+poonceonces 13.48Thu Jul 17 18:12:51 UTC
Now the second write and the first read is converted to
store_releaseandload_acquireand it gives us the following litmus test[2]C MP+pooncerelease+poacquireonce
(*
* Result: Never
*
* This litmus test demonstrates thatsmp_store_release()and
*smp_load_acquire()provide sufficient ordering for the message-passing
* pattern.
*){}
P0(int *buf, int *flag){ WRITE_ONCE(*buf, 1); smp_store_release(flag, 1);}
P1(int *buf, int *flag){ int r0; int r1; r0 = smp_load_acquire(flag); r1 = READ_ONCE(*buf);}
exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)
Running the generated bpf program shows that the bad outcome is *not* possible on powerpc with the implementation in this patch:
[fedora@linux-kernel blitmus]$ sudo ./mp_pooncerelease_poacquireonce Starting litmus test with configuration:
Test: MP+pooncerelease+poacquireonce Iterations: 4100Test MP+pooncerelease+poacquireonce Allowed
Histogram (3 states)
21036021 :>1:r0=0; 1:r1=0;
14488694 :>1:r0=0; 1:r1=1;
5475285 :>1:r0=1; 1:r1=1;
NoWitnesses
Positive: 0, Negative: 41000000
Condition exists (1:r0=1 /\ 1:r1=0) is NOT validated Observation MP+pooncerelease+poacquireonce Never 0 41000000 Time MP+pooncerelease+poacquireonce 13.74
Quoted 5 lines
Thu Jul 17 18:13:40 UTC
[0] https://github.com/puranjaymohan/blitmus
[1] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpoonceonces.litmus
[2] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpooncerelease%2Bpoacquireonce.litmus
Hi Puranjay,
Thanks for the patch. I applied the patch and tested it.
Before this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #528/1 verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK #528 verifier_load_acquire:OK #565/1 verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK #565 verifier_store_release:OK Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILED
After this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #529/1 verifier_load_acquire/load-acquire, 8-bit:OK #529/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #529/3 verifier_load_acquire/load-acquire, 16-bit:OK #529/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #529/5 verifier_load_acquire/load-acquire, 32-bit:OK #529/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #529/7 verifier_load_acquire/load-acquire, 64-bit:OK #529/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #529/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #529/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #529/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #529/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #529/13 verifier_load_acquire/misaligned load-acquire:OK #529/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #529/15 verifier_load_acquire/load-acquire from ctx pointer:OK #529/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #529/17 verifier_load_acquire/load-acquire with invalid register R15:OK #529/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #529/19 verifier_load_acquire/load-acquire from pkt pointer:OK #529/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #529/21 verifier_load_acquire/load-acquire from sock pointer:OK #529 verifier_load_acquire:OK #566/1 verifier_store_release/store-release, 8-bit:OK #566/2 verifier_store_release/store-release, 8-bit @unpriv:OK #566/3 verifier_store_release/store-release, 16-bit:OK #566/4 verifier_store_release/store-release, 16-bit @unpriv:OK #566/5 verifier_store_release/store-release, 32-bit:OK #566/6 verifier_store_release/store-release, 32-bit @unpriv:OK #566/7 verifier_store_release/store-release, 64-bit:OK #566/8 verifier_store_release/store-release, 64-bit @unpriv:OK #566/9 verifier_store_release/store-release with uninitialized src_reg:OK #566/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #566/11 verifier_store_release/store-release with uninitialized dst_reg:OK #566/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #566/13 verifier_store_release/store-release with non-pointer dst_reg:OK #566/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #566/15 verifier_store_release/misaligned store-release:OK #566/16 verifier_store_release/misaligned store-release @unpriv:OK #566/17 verifier_store_release/store-release to ctx pointer:OK #566/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #566/19 verifier_store_release/store-release, leak pointer to stack:OK #566/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #566/21 verifier_store_release/store-release, leak pointer to map:OK #566/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #566/23 verifier_store_release/store-release with invalid register R15:OK #566/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #566/25 verifier_store_release/store-release to pkt pointer:OK #566/26 verifier_store_release/store-release to flow_keys pointer:OK #566/27 verifier_store_release/store-release to sock pointer:OK #566 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Regards,
Saket
Daniel Borkmann
maintainerOn 7/24/25 12:27 PM, Saket Kumar Bhaskar wrote:
[...]
Quoted 84 lines
Thanks for the patch. I applied the patch and tested it.
Before this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #528/1 verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK #528 verifier_load_acquire:OK #565/1 verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK #565 verifier_store_release:OK Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILEDAfter this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #529/1 verifier_load_acquire/load-acquire, 8-bit:OK #529/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #529/3 verifier_load_acquire/load-acquire, 16-bit:OK #529/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #529/5 verifier_load_acquire/load-acquire, 32-bit:OK #529/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #529/7 verifier_load_acquire/load-acquire, 64-bit:OK #529/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #529/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #529/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #529/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #529/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #529/13 verifier_load_acquire/misaligned load-acquire:OK #529/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #529/15 verifier_load_acquire/load-acquire from ctx pointer:OK #529/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #529/17 verifier_load_acquire/load-acquire with invalid register R15:OK #529/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #529/19 verifier_load_acquire/load-acquire from pkt pointer:OK #529/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #529/21 verifier_load_acquire/load-acquire from sock pointer:OK #529 verifier_load_acquire:OK #566/1 verifier_store_release/store-release, 8-bit:OK #566/2 verifier_store_release/store-release, 8-bit @unpriv:OK #566/3 verifier_store_release/store-release, 16-bit:OK #566/4 verifier_store_release/store-release, 16-bit @unpriv:OK #566/5 verifier_store_release/store-release, 32-bit:OK #566/6 verifier_store_release/store-release, 32-bit @unpriv:OK #566/7 verifier_store_release/store-release, 64-bit:OK #566/8 verifier_store_release/store-release, 64-bit @unpriv:OK #566/9 verifier_store_release/store-release with uninitialized src_reg:OK #566/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #566/11 verifier_store_release/store-release with uninitialized dst_reg:OK #566/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #566/13 verifier_store_release/store-release with non-pointer dst_reg:OK #566/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #566/15 verifier_store_release/misaligned store-release:OK #566/16 verifier_store_release/misaligned store-release @unpriv:OK #566/17 verifier_store_release/store-release to ctx pointer:OK #566/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #566/19 verifier_store_release/store-release, leak pointer to stack:OK #566/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #566/21 verifier_store_release/store-release, leak pointer to map:OK #566/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #566/23 verifier_store_release/store-release with invalid register R15:OK #566/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #566/25 verifier_store_release/store-release to pkt pointer:OK #566/26 verifier_store_release/store-release to flow_keys pointer:OK #566/27 verifier_store_release/store-release to sock pointer:OK #566 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Quoted 1 line
Tested-by Saket Kumar Bhaskar <[email protected]>
Michael/Madhavan, I presume you'll pick this patch up?
Thanks,
Daniel
Madhavan Srinivasan
maintainerQuoted 86 lines · Daniel Borkmann
On 7/24/25 12:27 PM, Saket Kumar Bhaskar wrote:
[...]Quoted 84 lines
Thanks for the patch. I applied the patch and tested it.
Before this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #528/1 verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK #528 verifier_load_acquire:OK #565/1 verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK #565 verifier_store_release:OK Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILEDAfter this patch:
# ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #529/1 verifier_load_acquire/load-acquire, 8-bit:OK #529/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #529/3 verifier_load_acquire/load-acquire, 16-bit:OK #529/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #529/5 verifier_load_acquire/load-acquire, 32-bit:OK #529/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #529/7 verifier_load_acquire/load-acquire, 64-bit:OK #529/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #529/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #529/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #529/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #529/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #529/13 verifier_load_acquire/misaligned load-acquire:OK #529/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #529/15 verifier_load_acquire/load-acquire from ctx pointer:OK #529/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #529/17 verifier_load_acquire/load-acquire with invalid register R15:OK #529/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #529/19 verifier_load_acquire/load-acquire from pkt pointer:OK #529/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #529/21 verifier_load_acquire/load-acquire from sock pointer:OK #529 verifier_load_acquire:OK #566/1 verifier_store_release/store-release, 8-bit:OK #566/2 verifier_store_release/store-release, 8-bit @unpriv:OK #566/3 verifier_store_release/store-release, 16-bit:OK #566/4 verifier_store_release/store-release, 16-bit @unpriv:OK #566/5 verifier_store_release/store-release, 32-bit:OK #566/6 verifier_store_release/store-release, 32-bit @unpriv:OK #566/7 verifier_store_release/store-release, 64-bit:OK #566/8 verifier_store_release/store-release, 64-bit @unpriv:OK #566/9 verifier_store_release/store-release with uninitialized src_reg:OK #566/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #566/11 verifier_store_release/store-release with uninitialized dst_reg:OK #566/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #566/13 verifier_store_release/store-release with non-pointer dst_reg:OK #566/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #566/15 verifier_store_release/misaligned store-release:OK #566/16 verifier_store_release/misaligned store-release @unpriv:OK #566/17 verifier_store_release/store-release to ctx pointer:OK #566/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #566/19 verifier_store_release/store-release, leak pointer to stack:OK #566/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #566/21 verifier_store_release/store-release, leak pointer to map:OK #566/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #566/23 verifier_store_release/store-release with invalid register R15:OK #566/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #566/25 verifier_store_release/store-release to pkt pointer:OK #566/26 verifier_store_release/store-release to flow_keys pointer:OK #566/27 verifier_store_release/store-release to sock pointer:OK #566 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Quoted 4 lines
Quoted 1 line
Tested-by Saket Kumar Bhaskar <[email protected]>Michael/Madhavan, I presume you'll pick this patch up?
yes, Will do.
Thanks
Maddy
Quoted 2 lines
Thanks,
Daniel
Quoted 76 lines · Puranjay Mohan
Add JIT support for the
load_acquireandstore_releaseinstructions. The implementation is similar to the kernel where:load_acquire => plain load -> lwsync store_release => lwsync -> plain storeTo test the correctness of the implementation, following selftests were run:
[fedora@linux-kernel bpf]$ sudo ./test_progs -a \ verifier_load_acquire,verifier_store_release,atomics #11/1 atomics/add:OK #11/2 atomics/sub:OK #11/3 atomics/and:OK #11/4 atomics/or:OK #11/5 atomics/xor:OK #11/6 atomics/cmpxchg:OK #11/7 atomics/xchg:OK #11 atomics:OK #519/1 verifier_load_acquire/load-acquire, 8-bit:OK #519/2 verifier_load_acquire/load-acquire, 8-bit @unpriv:OK #519/3 verifier_load_acquire/load-acquire, 16-bit:OK #519/4 verifier_load_acquire/load-acquire, 16-bit @unpriv:OK #519/5 verifier_load_acquire/load-acquire, 32-bit:OK #519/6 verifier_load_acquire/load-acquire, 32-bit @unpriv:OK #519/7 verifier_load_acquire/load-acquire, 64-bit:OK #519/8 verifier_load_acquire/load-acquire, 64-bit @unpriv:OK #519/9 verifier_load_acquire/load-acquire with uninitialized src_reg:OK #519/10 verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK #519/11 verifier_load_acquire/load-acquire with non-pointer src_reg:OK #519/12 verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK #519/13 verifier_load_acquire/misaligned load-acquire:OK #519/14 verifier_load_acquire/misaligned load-acquire @unpriv:OK #519/15 verifier_load_acquire/load-acquire from ctx pointer:OK #519/16 verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK #519/17 verifier_load_acquire/load-acquire with invalid register R15:OK #519/18 verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK #519/19 verifier_load_acquire/load-acquire from pkt pointer:OK #519/20 verifier_load_acquire/load-acquire from flow_keys pointer:OK #519/21 verifier_load_acquire/load-acquire from sock pointer:OK #519 verifier_load_acquire:OK #556/1 verifier_store_release/store-release, 8-bit:OK #556/2 verifier_store_release/store-release, 8-bit @unpriv:OK #556/3 verifier_store_release/store-release, 16-bit:OK #556/4 verifier_store_release/store-release, 16-bit @unpriv:OK #556/5 verifier_store_release/store-release, 32-bit:OK #556/6 verifier_store_release/store-release, 32-bit @unpriv:OK #556/7 verifier_store_release/store-release, 64-bit:OK #556/8 verifier_store_release/store-release, 64-bit @unpriv:OK #556/9 verifier_store_release/store-release with uninitialized src_reg:OK #556/10 verifier_store_release/store-release with uninitialized src_reg @unpriv:OK #556/11 verifier_store_release/store-release with uninitialized dst_reg:OK #556/12 verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK #556/13 verifier_store_release/store-release with non-pointer dst_reg:OK #556/14 verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK #556/15 verifier_store_release/misaligned store-release:OK #556/16 verifier_store_release/misaligned store-release @unpriv:OK #556/17 verifier_store_release/store-release to ctx pointer:OK #556/18 verifier_store_release/store-release to ctx pointer @unpriv:OK #556/19 verifier_store_release/store-release, leak pointer to stack:OK #556/20 verifier_store_release/store-release, leak pointer to stack @unpriv:OK #556/21 verifier_store_release/store-release, leak pointer to map:OK #556/22 verifier_store_release/store-release, leak pointer to map @unpriv:OK #556/23 verifier_store_release/store-release with invalid register
Quoted 9 lines
R15:OK #556/24 verifier_store_release/store-release with invalid register R15 @unpriv:OK #556/25 verifier_store_release/store-release to pkt pointer:OK #556/26 verifier_store_release/store-release to flow_keys pointer:OK #556/27 verifier_store_release/store-release to sock pointer:OK #556 verifier_store_release:OK Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Thanks for the patch. Looks good.
Quoted 135 lines
Signed-off-by Puranjay Mohan <[email protected]>--- arch/powerpc/include/asm/ppc-opcode.h | 1 + arch/powerpc/net/bpf_jit_comp64.c | 82 ++++++++++++++++++++ tools/testing/selftests/bpf/progs/bpf_misc.h | 3 +- 3 files changed, 85 insertions(+), 1 deletion(-)arch/powerpc/include/asm/ppc-opcode.h +1
-0@@ -425,6 +425,7 @@#define PPC_RAW_SC () (0x44000002)#define PPC_RAW_SYNC () (0x7c0004ac)#define PPC_RAW_ISYNC () (0x4c00012c) +#define PPC_RAW_LWSYNC () (0x7c2004ac)/* * Define what the VSX XX1 form instructions will look like, then add arch/powerpc/net/bpf_jit_comp64.c +82
-0@@ -409,6 +409,71 @@ asm (" blr ;" ); +static int emit_atomic_ld_st (const struct bpf_insn insn,struct codegen_context *ctx,u32 *image) +{ +u32 code = insn.code; +u32 dst_reg =bpf_to_ppc (insn.dst_reg); +u32 src_reg =bpf_to_ppc (insn.src_reg); +u32 size =BPF_SIZE (code); +u32 tmp1_reg =bpf_to_ppc (TMP_REG_1 ); +u32 tmp2_reg =bpf_to_ppc (TMP_REG_2 ); +s16 off = insn.off; +s32 imm = insn.imm; + +switch (imm) { +case BPF_LOAD_ACQ : +switch (size) { +case BPF_B : +EMIT (PPC_RAW_LBZ (dst_reg, src_reg, off)); +break ; +case BPF_H : +EMIT (PPC_RAW_LHZ (dst_reg, src_reg, off)); +break ; +case BPF_W : +EMIT (PPC_RAW_LWZ (dst_reg, src_reg, off)); +break ; +case BPF_DW : +if (off %4 ) { +EMIT (PPC_RAW_LI (tmp1_reg, off)); +EMIT (PPC_RAW_LDX (dst_reg, src_reg, tmp1_reg)); + }else { +EMIT (PPC_RAW_LD (dst_reg, src_reg, off)); + } +break ; + } +EMIT (PPC_RAW_LWSYNC ()); +break ; +case BPF_STORE_REL : +EMIT (PPC_RAW_LWSYNC ()); +switch (size) { +case BPF_B : +EMIT (PPC_RAW_STB (src_reg, dst_reg, off)); +break ; +case BPF_H : +EMIT (PPC_RAW_STH (src_reg, dst_reg, off)); +break ; +case BPF_W : +EMIT (PPC_RAW_STW (src_reg, dst_reg, off)); +break ; +case BPF_DW : +if (off %4 ) { +EMIT (PPC_RAW_LI (tmp2_reg, off)); +EMIT (PPC_RAW_STDX (src_reg, dst_reg, tmp2_reg)); + }else { +EMIT (PPC_RAW_STD (src_reg, dst_reg, off)); + } +break ; + } +break ; +default : +pr_err_ratelimited ("unexpected atomic load/store op code %02x\n" , + imm); +return -EINVAL ; + } + +return 0 ; +} +/* Assemble the body code between the prologue & epilogue */ int bpf_jit_build_body (struct bpf_prog *fp,u32 *image,u32 *fimage,struct codegen_context *ctx,u32 *addrs,int pass,bool extra_pass)@@ -898,8 +963,25 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code/* * BPF_STX ATOMIC (atomic ops) */ +case BPF_STX |BPF_ATOMIC |BPF_B : +case BPF_STX |BPF_ATOMIC |BPF_H :case BPF_STX |BPF_ATOMIC |BPF_W :case BPF_STX |BPF_ATOMIC |BPF_DW : +if (bpf_atomic_is_load_store (&insn[i])) { + ret =emit_atomic_ld_st (insn[i], ctx, image); +if (ret) +return ret; + +if (size !=BPF_DW &&insn_is_zext (&insn[i +1 ])) + addrs[++i] = ctx->idx *4 ; +break ; + }else if (size ==BPF_B || size ==BPF_H ) { +pr_err_ratelimited ( +"eBPF filter atomic op code %02x (@%d) unsupported\n" , + code, i); +return -EOPNOTSUPP ; + } + save_reg = tmp2_reg; ret_reg = src_reg;tools/testing/selftests/bpf/progs/bpf_misc.h +2
-1@@ -229,7 +229,8 @@#if __clang_major__ >=18 && defined(ENABLE_ATOMICS_TESTS ) && \ (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 )) + (defined(__TARGET_ARCH_riscv) && __riscv_xlen ==64 )) || \ + (defined(__TARGET_ARCH_powerpc))#define CAN_USE_LOAD_ACQ_STORE_REL #endif