Skip to contentSkip to conversation

powerpc64/bpf: Add jit support for load_acquire and store_release

PATCH RESEND In review Puranjay Mohan 5 participants 6 messages Jul 17, 2025 – Jul 28, 2025 1 message not in the archive

Subsystems (6)
PowerPC Kselftest BPF BPF selftests +2 more subsystems
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH RESEND 1/1 powerpc64/bpf: Add jit support for load_acquire and store_release

Add JIT support for the load_acquire and store_release instructions. The implementation is similar to the kernel where:

        load_acquire  => plain load -> lwsync
        store_release => lwsync -> plain store

To test the correctness of the implementation, following selftests were run:

  [fedora@linux-kernel bpf]$ sudo ./test_progs -a \
  verifier_load_acquire,verifier_store_release,atomics
  #11/1    atomics/add:OK
  #11/2    atomics/sub:OK
  #11/3    atomics/and:OK
  #11/4    atomics/or:OK
  #11/5    atomics/xor:OK
  #11/6    atomics/cmpxchg:OK
  #11/7    atomics/xchg:OK
  #11      atomics:OK
  #519/1   verifier_load_acquire/load-acquire, 8-bit:OK
  #519/2   verifier_load_acquire/load-acquire, 8-bit @unpriv:OK
  #519/3   verifier_load_acquire/load-acquire, 16-bit:OK
  #519/4   verifier_load_acquire/load-acquire, 16-bit @unpriv:OK
  #519/5   verifier_load_acquire/load-acquire, 32-bit:OK
  #519/6   verifier_load_acquire/load-acquire, 32-bit @unpriv:OK
  #519/7   verifier_load_acquire/load-acquire, 64-bit:OK
  #519/8   verifier_load_acquire/load-acquire, 64-bit @unpriv:OK
  #519/9   verifier_load_acquire/load-acquire with uninitialized
  src_reg:OK
  #519/10  verifier_load_acquire/load-acquire with uninitialized src_reg
  @unpriv:OK
  #519/11  verifier_load_acquire/load-acquire with non-pointer src_reg:OK
  #519/12  verifier_load_acquire/load-acquire with non-pointer src_reg
  @unpriv:OK
  #519/13  verifier_load_acquire/misaligned load-acquire:OK
  #519/14  verifier_load_acquire/misaligned load-acquire @unpriv:OK
  #519/15  verifier_load_acquire/load-acquire from ctx pointer:OK
  #519/16  verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK
  #519/17  verifier_load_acquire/load-acquire with invalid register R15:OK
  #519/18  verifier_load_acquire/load-acquire with invalid register R15
  @unpriv:OK
  #519/19  verifier_load_acquire/load-acquire from pkt pointer:OK
  #519/20  verifier_load_acquire/load-acquire from flow_keys pointer:OK
  #519/21  verifier_load_acquire/load-acquire from sock pointer:OK
  #519     verifier_load_acquire:OK
  #556/1   verifier_store_release/store-release, 8-bit:OK
  #556/2   verifier_store_release/store-release, 8-bit @unpriv:OK
  #556/3   verifier_store_release/store-release, 16-bit:OK
  #556/4   verifier_store_release/store-release, 16-bit @unpriv:OK
  #556/5   verifier_store_release/store-release, 32-bit:OK
  #556/6   verifier_store_release/store-release, 32-bit @unpriv:OK
  #556/7   verifier_store_release/store-release, 64-bit:OK
  #556/8   verifier_store_release/store-release, 64-bit @unpriv:OK
  #556/9   verifier_store_release/store-release with uninitialized
  src_reg:OK
  #556/10  verifier_store_release/store-release with uninitialized src_reg
  @unpriv:OK
  #556/11  verifier_store_release/store-release with uninitialized
  dst_reg:OK
  #556/12  verifier_store_release/store-release with uninitialized dst_reg
  @unpriv:OK
  #556/13  verifier_store_release/store-release with non-pointer
  dst_reg:OK
  #556/14  verifier_store_release/store-release with non-pointer dst_reg
  @unpriv:OK
  #556/15  verifier_store_release/misaligned store-release:OK
  #556/16  verifier_store_release/misaligned store-release @unpriv:OK
  #556/17  verifier_store_release/store-release to ctx pointer:OK
  #556/18  verifier_store_release/store-release to ctx pointer @unpriv:OK
  #556/19  verifier_store_release/store-release, leak pointer to stack:OK
  #556/20  verifier_store_release/store-release, leak pointer to stack
  @unpriv:OK
  #556/21  verifier_store_release/store-release, leak pointer to map:OK
  #556/22  verifier_store_release/store-release, leak pointer to map
  @unpriv:OK
  #556/23  verifier_store_release/store-release with invalid register
  R15:OK
  #556/24  verifier_store_release/store-release with invalid register R15
  @unpriv:OK
  #556/25  verifier_store_release/store-release to pkt pointer:OK
  #556/26  verifier_store_release/store-release to flow_keys pointer:OK
  #556/27  verifier_store_release/store-release to sock pointer:OK
  #556     verifier_store_release:OK
  Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
---
 arch/powerpc/include/asm/ppc-opcode.h        |  1 +
 arch/powerpc/net/bpf_jit_comp64.c            | 82 ++++++++++++++++++++
 tools/testing/selftests/bpf/progs/bpf_misc.h |  3 +-
 3 files changed, 85 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/ppc-opcode.h +1 -0
index 4312bcb913a42..8053b24afc395 100644
@@ -425,6 +425,7 @@
 #define PPC_RAW_SC()			(0x44000002)
 #define PPC_RAW_SYNC()			(0x7c0004ac)
 #define PPC_RAW_ISYNC()			(0x4c00012c)
+#define PPC_RAW_LWSYNC()		(0x7c2004ac)
 
 /*
  * Define what the VSX XX1 form instructions will look like, then add
arch/powerpc/net/bpf_jit_comp64.c +82 -0

96 lines not on this page · raw message

tools/testing/selftests/bpf/progs/bpf_misc.h +2 -1
index 530752ddde8e4..c1cfd297aabf1 100644
@@ -229,7 +229,8 @@
 
 #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) &&		\
 	(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) ||	\
-	 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
+	 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) || \
+	  (defined(__TARGET_ARCH_powerpc))
 #define CAN_USE_LOAD_ACQ_STORE_REL
 #endif
 
Signature · 1 line
2.47.1
#2 lore raw

Puranjay Mohan <[email protected]> writes:

Somehow the cover letter for this patch was missed, adding it here:

To test the functionality of these special instructions, a tool called blitmus[0] was used to convert the following baseline litmus test[1] to bpf programs:

C MP+poonceonces

(*
* Result: Sometimes
*
* Can the counter-intuitive message-passing outcome be prevented with
* no ordering at all?
*)

{}

P0(int *buf, int *flag)

 {
         WRITE_ONCE(*buf, 1);
         WRITE_ONCE(*flag, 1);

}

P1(int *buf, int *flag)

 {
         int r0;
         int r1;

         r0 = READ_ONCE(*flag);
         r1 = READ_ONCE(*buf);

}

exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)

Running the generated bpf program shows that the bad outcome is possible on powerpc:

[fedora@linux-kernel blitmus]$ sudo ./mp_poonceonces Starting litmus test with configuration:
Test: MP+poonceonces
Iterations: 4100

Test MP+poonceonces Allowed
Histogram (4 states)
21548375 :>1:r0=0; 1:r1=0;
301187 :>1:r0=0; 1:r1=1;
337147 *>1:r0=1; 1:r1=0;
18813291 :>1:r0=1; 1:r1=1;
Ok

Witnesses
Positive: 337147, Negative: 40662853 Condition exists (1:r0=1 /\ 1:r1=0) is validated Observation MP+poonceonces Sometimes 337147 40662853 Time MP+poonceonces 13.48

Thu Jul 17 18:12:51 UTC

Now the second write and the first read is converted to store_release and load_acquire and it gives us the following litmus test[2]

C MP+pooncerelease+poacquireonce

(*
* Result: Never
*
* This litmus test demonstrates that smp_store_release() and
* smp_load_acquire() provide sufficient ordering for the message-passing
* pattern.
*)

{}

P0(int *buf, int *flag)

 {
         WRITE_ONCE(*buf, 1);
         smp_store_release(flag, 1);

}

P1(int *buf, int *flag)

 {
         int r0;
         int r1;

         r0 = smp_load_acquire(flag);
         r1 = READ_ONCE(*buf);

}

exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)

Running the generated bpf program shows that the bad outcome is *not* possible on powerpc with the implementation in this patch:

[fedora@linux-kernel blitmus]$ sudo ./mp_pooncerelease_poacquireonce Starting litmus test with configuration:
Test: MP+pooncerelease+poacquireonce
Iterations: 4100

Test MP+pooncerelease+poacquireonce Allowed
Histogram (3 states)
21036021 :>1:r0=0; 1:r1=0;
14488694 :>1:r0=0; 1:r1=1;
5475285 :>1:r0=1; 1:r1=1;
No

Witnesses
Positive: 0, Negative: 41000000
Condition exists (1:r0=1 /\ 1:r1=0) is NOT validated Observation MP+pooncerelease+poacquireonce Never 0 41000000 Time MP+pooncerelease+poacquireonce 13.74

Thu Jul 17 18:13:40 UTC

[0] https://github.com/puranjaymohan/blitmus
[1] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpoonceonces.litmus
[2] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpooncerelease%2Bpoacquireonce.litmus

#3 lore raw
Quoted 114 lines · On Thu, Jul 17, 2025 at 08:56:45PM +0000, [email protected] wrote:

Puranjay Mohan <[email protected]> writes:

Somehow the cover letter for this patch was missed, adding it here:

To test the functionality of these special instructions, a tool called blitmus[0] was used to convert the following baseline litmus test[1] to bpf programs:

C MP+poonceonces

(*
* Result: Sometimes
*
* Can the counter-intuitive message-passing outcome be prevented with
* no ordering at all?
*)

{}

P0(int *buf, int *flag)

 {
         WRITE_ONCE(*buf, 1);
         WRITE_ONCE(*flag, 1);

}

P1(int *buf, int *flag)

 {
         int r0;
         int r1;

         r0 = READ_ONCE(*flag);
         r1 = READ_ONCE(*buf);

}

exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)

Running the generated bpf program shows that the bad outcome is possible on powerpc:

[fedora@linux-kernel blitmus]$ sudo ./mp_poonceonces Starting litmus test with configuration:

   Test: MP+poonceonces
   Iterations: 4100

Test MP+poonceonces Allowed
Histogram (4 states)
21548375 :>1:r0=0; 1:r1=0;
301187 :>1:r0=0; 1:r1=1;
337147 *>1:r0=1; 1:r1=0;
18813291 :>1:r0=1; 1:r1=1;
Ok

Witnesses
Positive: 337147, Negative: 40662853 Condition exists (1:r0=1 /\ 1:r1=0) is validated Observation MP+poonceonces Sometimes 337147 40662853 Time MP+poonceonces 13.48

Thu Jul 17 18:12:51 UTC

Now the second write and the first read is converted to store_release and load_acquire and it gives us the following litmus test[2]

C MP+pooncerelease+poacquireonce

(*
* Result: Never
*
* This litmus test demonstrates that smp_store_release() and
* smp_load_acquire() provide sufficient ordering for the message-passing
* pattern.
*)

{}

P0(int *buf, int *flag)

 {
         WRITE_ONCE(*buf, 1);
         smp_store_release(flag, 1);

}

P1(int *buf, int *flag)

 {
         int r0;
         int r1;

         r0 = smp_load_acquire(flag);
         r1 = READ_ONCE(*buf);

}

exists (1:r0=1 /\ 1:r1=0) (* Bad outcome. *)

Running the generated bpf program shows that the bad outcome is *not* possible on powerpc with the implementation in this patch:

[fedora@linux-kernel blitmus]$ sudo ./mp_pooncerelease_poacquireonce Starting litmus test with configuration:

   Test: MP+pooncerelease+poacquireonce
   Iterations: 4100

Test MP+pooncerelease+poacquireonce Allowed
Histogram (3 states)
21036021 :>1:r0=0; 1:r1=0;
14488694 :>1:r0=0; 1:r1=1;
5475285 :>1:r0=1; 1:r1=1;
No

Witnesses
Positive: 0, Negative: 41000000
Condition exists (1:r0=1 /\ 1:r1=0) is NOT validated Observation MP+pooncerelease+poacquireonce Never 0 41000000 Time MP+pooncerelease+poacquireonce 13.74

Quoted 5 lines

Thu Jul 17 18:13:40 UTC

[0] https://github.com/puranjaymohan/blitmus
[1] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpoonceonces.litmus
[2] https://github.com/puranjaymohan/blitmus/blob/main/litmus_tests/MP%2Bpooncerelease%2Bpoacquireonce.litmus

Hi Puranjay,

Thanks for the patch. I applied the patch and tested it.

Before this patch:

# ./test_progs -a \
  verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#528/1   verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK
#528     verifier_load_acquire:OK
#565/1   verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK
#565     verifier_store_release:OK
Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILED

After this patch:

# ./test_progs -a \
  verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#529/1   verifier_load_acquire/load-acquire, 8-bit:OK
#529/2   verifier_load_acquire/load-acquire, 8-bit @unpriv:OK
#529/3   verifier_load_acquire/load-acquire, 16-bit:OK
#529/4   verifier_load_acquire/load-acquire, 16-bit @unpriv:OK
#529/5   verifier_load_acquire/load-acquire, 32-bit:OK
#529/6   verifier_load_acquire/load-acquire, 32-bit @unpriv:OK
#529/7   verifier_load_acquire/load-acquire, 64-bit:OK
#529/8   verifier_load_acquire/load-acquire, 64-bit @unpriv:OK
#529/9   verifier_load_acquire/load-acquire with uninitialized src_reg:OK
#529/10  verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK
#529/11  verifier_load_acquire/load-acquire with non-pointer src_reg:OK
#529/12  verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK
#529/13  verifier_load_acquire/misaligned load-acquire:OK
#529/14  verifier_load_acquire/misaligned load-acquire @unpriv:OK
#529/15  verifier_load_acquire/load-acquire from ctx pointer:OK
#529/16  verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK
#529/17  verifier_load_acquire/load-acquire with invalid register R15:OK
#529/18  verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK
#529/19  verifier_load_acquire/load-acquire from pkt pointer:OK
#529/20  verifier_load_acquire/load-acquire from flow_keys pointer:OK
#529/21  verifier_load_acquire/load-acquire from sock pointer:OK
#529     verifier_load_acquire:OK
#566/1   verifier_store_release/store-release, 8-bit:OK
#566/2   verifier_store_release/store-release, 8-bit @unpriv:OK
#566/3   verifier_store_release/store-release, 16-bit:OK
#566/4   verifier_store_release/store-release, 16-bit @unpriv:OK
#566/5   verifier_store_release/store-release, 32-bit:OK
#566/6   verifier_store_release/store-release, 32-bit @unpriv:OK
#566/7   verifier_store_release/store-release, 64-bit:OK
#566/8   verifier_store_release/store-release, 64-bit @unpriv:OK
#566/9   verifier_store_release/store-release with uninitialized src_reg:OK
#566/10  verifier_store_release/store-release with uninitialized src_reg @unpriv:OK
#566/11  verifier_store_release/store-release with uninitialized dst_reg:OK
#566/12  verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK
#566/13  verifier_store_release/store-release with non-pointer dst_reg:OK
#566/14  verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK
#566/15  verifier_store_release/misaligned store-release:OK
#566/16  verifier_store_release/misaligned store-release @unpriv:OK
#566/17  verifier_store_release/store-release to ctx pointer:OK
#566/18  verifier_store_release/store-release to ctx pointer @unpriv:OK
#566/19  verifier_store_release/store-release, leak pointer to stack:OK
#566/20  verifier_store_release/store-release, leak pointer to stack @unpriv:OK
#566/21  verifier_store_release/store-release, leak pointer to map:OK
#566/22  verifier_store_release/store-release, leak pointer to map @unpriv:OK
#566/23  verifier_store_release/store-release with invalid register R15:OK
#566/24  verifier_store_release/store-release with invalid register R15 @unpriv:OK
#566/25  verifier_store_release/store-release to pkt pointer:OK
#566/26  verifier_store_release/store-release to flow_keys pointer:OK
#566/27  verifier_store_release/store-release to sock pointer:OK
#566     verifier_store_release:OK
Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED

Regards,
Saket

maintainer
#4 lore raw

On 7/24/25 12:27 PM, Saket Kumar Bhaskar wrote:
[...]

Quoted 84 lines

Thanks for the patch. I applied the patch and tested it.

Before this patch:

# ./test_progs -a \
   verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#528/1   verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK
#528     verifier_load_acquire:OK
#565/1   verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK
#565     verifier_store_release:OK
Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILED

After this patch:

# ./test_progs -a \
   verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#529/1   verifier_load_acquire/load-acquire, 8-bit:OK
#529/2   verifier_load_acquire/load-acquire, 8-bit @unpriv:OK
#529/3   verifier_load_acquire/load-acquire, 16-bit:OK
#529/4   verifier_load_acquire/load-acquire, 16-bit @unpriv:OK
#529/5   verifier_load_acquire/load-acquire, 32-bit:OK
#529/6   verifier_load_acquire/load-acquire, 32-bit @unpriv:OK
#529/7   verifier_load_acquire/load-acquire, 64-bit:OK
#529/8   verifier_load_acquire/load-acquire, 64-bit @unpriv:OK
#529/9   verifier_load_acquire/load-acquire with uninitialized src_reg:OK
#529/10  verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK
#529/11  verifier_load_acquire/load-acquire with non-pointer src_reg:OK
#529/12  verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK
#529/13  verifier_load_acquire/misaligned load-acquire:OK
#529/14  verifier_load_acquire/misaligned load-acquire @unpriv:OK
#529/15  verifier_load_acquire/load-acquire from ctx pointer:OK
#529/16  verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK
#529/17  verifier_load_acquire/load-acquire with invalid register R15:OK
#529/18  verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK
#529/19  verifier_load_acquire/load-acquire from pkt pointer:OK
#529/20  verifier_load_acquire/load-acquire from flow_keys pointer:OK
#529/21  verifier_load_acquire/load-acquire from sock pointer:OK
#529     verifier_load_acquire:OK
#566/1   verifier_store_release/store-release, 8-bit:OK
#566/2   verifier_store_release/store-release, 8-bit @unpriv:OK
#566/3   verifier_store_release/store-release, 16-bit:OK
#566/4   verifier_store_release/store-release, 16-bit @unpriv:OK
#566/5   verifier_store_release/store-release, 32-bit:OK
#566/6   verifier_store_release/store-release, 32-bit @unpriv:OK
#566/7   verifier_store_release/store-release, 64-bit:OK
#566/8   verifier_store_release/store-release, 64-bit @unpriv:OK
#566/9   verifier_store_release/store-release with uninitialized src_reg:OK
#566/10  verifier_store_release/store-release with uninitialized src_reg @unpriv:OK
#566/11  verifier_store_release/store-release with uninitialized dst_reg:OK
#566/12  verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK
#566/13  verifier_store_release/store-release with non-pointer dst_reg:OK
#566/14  verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK
#566/15  verifier_store_release/misaligned store-release:OK
#566/16  verifier_store_release/misaligned store-release @unpriv:OK
#566/17  verifier_store_release/store-release to ctx pointer:OK
#566/18  verifier_store_release/store-release to ctx pointer @unpriv:OK
#566/19  verifier_store_release/store-release, leak pointer to stack:OK
#566/20  verifier_store_release/store-release, leak pointer to stack @unpriv:OK
#566/21  verifier_store_release/store-release, leak pointer to map:OK
#566/22  verifier_store_release/store-release, leak pointer to map @unpriv:OK
#566/23  verifier_store_release/store-release with invalid register R15:OK
#566/24  verifier_store_release/store-release with invalid register R15 @unpriv:OK
#566/25  verifier_store_release/store-release to pkt pointer:OK
#566/26  verifier_store_release/store-release to flow_keys pointer:OK
#566/27  verifier_store_release/store-release to sock pointer:OK
#566     verifier_store_release:OK
Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Quoted 1 line

Michael/Madhavan, I presume you'll pick this patch up?

Thanks,
Daniel

#5 lore raw
Quoted 86 lines · Daniel Borkmann

On 7/24/25 12:27 PM, Saket Kumar Bhaskar wrote:
[...]

Quoted 84 lines

Thanks for the patch. I applied the patch and tested it.

Before this patch:

# ./test_progs -a \
   verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#528/1   verifier_load_acquire/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test:OK
#528     verifier_load_acquire:OK
#565/1   verifier_store_release/Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test:OK
#565     verifier_store_release:OK
Summary: 3/9 PASSED, 0 SKIPPED, 0 FAILED

After this patch:

# ./test_progs -a \
   verifier_load_acquire,verifier_store_release,atomics
#11/1    atomics/add:OK
#11/2    atomics/sub:OK
#11/3    atomics/and:OK
#11/4    atomics/or:OK
#11/5    atomics/xor:OK
#11/6    atomics/cmpxchg:OK
#11/7    atomics/xchg:OK
#11      atomics:OK
#529/1   verifier_load_acquire/load-acquire, 8-bit:OK
#529/2   verifier_load_acquire/load-acquire, 8-bit @unpriv:OK
#529/3   verifier_load_acquire/load-acquire, 16-bit:OK
#529/4   verifier_load_acquire/load-acquire, 16-bit @unpriv:OK
#529/5   verifier_load_acquire/load-acquire, 32-bit:OK
#529/6   verifier_load_acquire/load-acquire, 32-bit @unpriv:OK
#529/7   verifier_load_acquire/load-acquire, 64-bit:OK
#529/8   verifier_load_acquire/load-acquire, 64-bit @unpriv:OK
#529/9   verifier_load_acquire/load-acquire with uninitialized src_reg:OK
#529/10  verifier_load_acquire/load-acquire with uninitialized src_reg @unpriv:OK
#529/11  verifier_load_acquire/load-acquire with non-pointer src_reg:OK
#529/12  verifier_load_acquire/load-acquire with non-pointer src_reg @unpriv:OK
#529/13  verifier_load_acquire/misaligned load-acquire:OK
#529/14  verifier_load_acquire/misaligned load-acquire @unpriv:OK
#529/15  verifier_load_acquire/load-acquire from ctx pointer:OK
#529/16  verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK
#529/17  verifier_load_acquire/load-acquire with invalid register R15:OK
#529/18  verifier_load_acquire/load-acquire with invalid register R15 @unpriv:OK
#529/19  verifier_load_acquire/load-acquire from pkt pointer:OK
#529/20  verifier_load_acquire/load-acquire from flow_keys pointer:OK
#529/21  verifier_load_acquire/load-acquire from sock pointer:OK
#529     verifier_load_acquire:OK
#566/1   verifier_store_release/store-release, 8-bit:OK
#566/2   verifier_store_release/store-release, 8-bit @unpriv:OK
#566/3   verifier_store_release/store-release, 16-bit:OK
#566/4   verifier_store_release/store-release, 16-bit @unpriv:OK
#566/5   verifier_store_release/store-release, 32-bit:OK
#566/6   verifier_store_release/store-release, 32-bit @unpriv:OK
#566/7   verifier_store_release/store-release, 64-bit:OK
#566/8   verifier_store_release/store-release, 64-bit @unpriv:OK
#566/9   verifier_store_release/store-release with uninitialized src_reg:OK
#566/10  verifier_store_release/store-release with uninitialized src_reg @unpriv:OK
#566/11  verifier_store_release/store-release with uninitialized dst_reg:OK
#566/12  verifier_store_release/store-release with uninitialized dst_reg @unpriv:OK
#566/13  verifier_store_release/store-release with non-pointer dst_reg:OK
#566/14  verifier_store_release/store-release with non-pointer dst_reg @unpriv:OK
#566/15  verifier_store_release/misaligned store-release:OK
#566/16  verifier_store_release/misaligned store-release @unpriv:OK
#566/17  verifier_store_release/store-release to ctx pointer:OK
#566/18  verifier_store_release/store-release to ctx pointer @unpriv:OK
#566/19  verifier_store_release/store-release, leak pointer to stack:OK
#566/20  verifier_store_release/store-release, leak pointer to stack @unpriv:OK
#566/21  verifier_store_release/store-release, leak pointer to map:OK
#566/22  verifier_store_release/store-release, leak pointer to map @unpriv:OK
#566/23  verifier_store_release/store-release with invalid register R15:OK
#566/24  verifier_store_release/store-release with invalid register R15 @unpriv:OK
#566/25  verifier_store_release/store-release to pkt pointer:OK
#566/26  verifier_store_release/store-release to flow_keys pointer:OK
#566/27  verifier_store_release/store-release to sock pointer:OK
#566     verifier_store_release:OK
Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED
Quoted 4 lines
Quoted 1 line

Michael/Madhavan, I presume you'll pick this patch up?

yes, Will do.

Thanks
Maddy

Quoted 2 lines

Thanks,
Daniel

maintainer Reviewed-by on v1 1/1
#6 lore raw
In reply to Puranjay Mohan
Quoted 76 lines · Puranjay Mohan

Add JIT support for the load_acquire and store_release instructions. The implementation is similar to the kernel where:

         load_acquire  => plain load -> lwsync
         store_release => lwsync -> plain store

To test the correctness of the implementation, following selftests were run:

   [fedora@linux-kernel bpf]$ sudo ./test_progs -a \
   verifier_load_acquire,verifier_store_release,atomics
   #11/1    atomics/add:OK
   #11/2    atomics/sub:OK
   #11/3    atomics/and:OK
   #11/4    atomics/or:OK
   #11/5    atomics/xor:OK
   #11/6    atomics/cmpxchg:OK
   #11/7    atomics/xchg:OK
   #11      atomics:OK
   #519/1   verifier_load_acquire/load-acquire, 8-bit:OK
   #519/2   verifier_load_acquire/load-acquire, 8-bit @unpriv:OK
   #519/3   verifier_load_acquire/load-acquire, 16-bit:OK
   #519/4   verifier_load_acquire/load-acquire, 16-bit @unpriv:OK
   #519/5   verifier_load_acquire/load-acquire, 32-bit:OK
   #519/6   verifier_load_acquire/load-acquire, 32-bit @unpriv:OK
   #519/7   verifier_load_acquire/load-acquire, 64-bit:OK
   #519/8   verifier_load_acquire/load-acquire, 64-bit @unpriv:OK
   #519/9   verifier_load_acquire/load-acquire with uninitialized
   src_reg:OK
   #519/10  verifier_load_acquire/load-acquire with uninitialized src_reg
   @unpriv:OK
   #519/11  verifier_load_acquire/load-acquire with non-pointer src_reg:OK
   #519/12  verifier_load_acquire/load-acquire with non-pointer src_reg
   @unpriv:OK
   #519/13  verifier_load_acquire/misaligned load-acquire:OK
   #519/14  verifier_load_acquire/misaligned load-acquire @unpriv:OK
   #519/15  verifier_load_acquire/load-acquire from ctx pointer:OK
   #519/16  verifier_load_acquire/load-acquire from ctx pointer @unpriv:OK
   #519/17  verifier_load_acquire/load-acquire with invalid register R15:OK
   #519/18  verifier_load_acquire/load-acquire with invalid register R15
   @unpriv:OK
   #519/19  verifier_load_acquire/load-acquire from pkt pointer:OK
   #519/20  verifier_load_acquire/load-acquire from flow_keys pointer:OK
   #519/21  verifier_load_acquire/load-acquire from sock pointer:OK
   #519     verifier_load_acquire:OK
   #556/1   verifier_store_release/store-release, 8-bit:OK
   #556/2   verifier_store_release/store-release, 8-bit @unpriv:OK
   #556/3   verifier_store_release/store-release, 16-bit:OK
   #556/4   verifier_store_release/store-release, 16-bit @unpriv:OK
   #556/5   verifier_store_release/store-release, 32-bit:OK
   #556/6   verifier_store_release/store-release, 32-bit @unpriv:OK
   #556/7   verifier_store_release/store-release, 64-bit:OK
   #556/8   verifier_store_release/store-release, 64-bit @unpriv:OK
   #556/9   verifier_store_release/store-release with uninitialized
   src_reg:OK
   #556/10  verifier_store_release/store-release with uninitialized src_reg
   @unpriv:OK
   #556/11  verifier_store_release/store-release with uninitialized
   dst_reg:OK
   #556/12  verifier_store_release/store-release with uninitialized dst_reg
   @unpriv:OK
   #556/13  verifier_store_release/store-release with non-pointer
   dst_reg:OK
   #556/14  verifier_store_release/store-release with non-pointer dst_reg
   @unpriv:OK
   #556/15  verifier_store_release/misaligned store-release:OK
   #556/16  verifier_store_release/misaligned store-release @unpriv:OK
   #556/17  verifier_store_release/store-release to ctx pointer:OK
   #556/18  verifier_store_release/store-release to ctx pointer @unpriv:OK
   #556/19  verifier_store_release/store-release, leak pointer to stack:OK
   #556/20  verifier_store_release/store-release, leak pointer to stack
   @unpriv:OK
   #556/21  verifier_store_release/store-release, leak pointer to map:OK
   #556/22  verifier_store_release/store-release, leak pointer to map
   @unpriv:OK
   #556/23  verifier_store_release/store-release with invalid register
Quoted 9 lines
   R15:OK
   #556/24  verifier_store_release/store-release with invalid register R15
   @unpriv:OK
   #556/25  verifier_store_release/store-release to pkt pointer:OK
   #556/26  verifier_store_release/store-release to flow_keys pointer:OK
   #556/27  verifier_store_release/store-release to sock pointer:OK
   #556     verifier_store_release:OK
   Summary: 3/55 PASSED, 0 SKIPPED, 0 FAILED

Thanks for the patch. Looks good.

Quoted 135 lines
---
  arch/powerpc/include/asm/ppc-opcode.h        |  1 +
  arch/powerpc/net/bpf_jit_comp64.c            | 82 ++++++++++++++++++++
  tools/testing/selftests/bpf/progs/bpf_misc.h |  3 +-
  3 files changed, 85 insertions(+), 1 deletion(-)
arch/powerpc/include/asm/ppc-opcode.h +1 -0
index 4312bcb913a42..8053b24afc395 100644
@@ -425,6 +425,7 @@
  #define PPC_RAW_SC()			(0x44000002)
  #define PPC_RAW_SYNC()			(0x7c0004ac)
  #define PPC_RAW_ISYNC()			(0x4c00012c)
+#define PPC_RAW_LWSYNC()		(0x7c2004ac)
 
  /*
   * Define what the VSX XX1 form instructions will look like, then add
arch/powerpc/net/bpf_jit_comp64.c +82 -0
index a25a6ffe7d7cc..025524378443e 100644
@@ -409,6 +409,71 @@ asm (
  "		blr				;"
  );
 
+static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image)
+{
+	u32 code = insn.code;
+	u32 dst_reg = bpf_to_ppc(insn.dst_reg);
+	u32 src_reg = bpf_to_ppc(insn.src_reg);
+	u32 size = BPF_SIZE(code);
+	u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
+	u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
+	s16 off = insn.off;
+	s32 imm = insn.imm;
+
+	switch (imm) {
+	case BPF_LOAD_ACQ:
+		switch (size) {
+		case BPF_B:
+			EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
+			break;
+		case BPF_H:
+			EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
+			break;
+		case BPF_W:
+			EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
+			break;
+		case BPF_DW:
+			if (off % 4) {
+				EMIT(PPC_RAW_LI(tmp1_reg, off));
+				EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg));
+			} else {
+				EMIT(PPC_RAW_LD(dst_reg, src_reg, off));
+			}
+			break;
+		}
+		EMIT(PPC_RAW_LWSYNC());
+		break;
+	case BPF_STORE_REL:
+		EMIT(PPC_RAW_LWSYNC());
+		switch (size) {
+		case BPF_B:
+			EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
+			break;
+		case BPF_H:
+			EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
+			break;
+		case BPF_W:
+			EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
+			break;
+		case BPF_DW:
+			if (off % 4) {
+				EMIT(PPC_RAW_LI(tmp2_reg, off));
+				EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg));
+			} else {
+				EMIT(PPC_RAW_STD(src_reg, dst_reg, off));
+			}
+			break;
+		}
+		break;
+	default:
+		pr_err_ratelimited("unexpected atomic load/store op code %02x\n",
+				   imm);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
  /* Assemble the body code between the prologue & epilogue */
  int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx,
  		       u32 *addrs, int pass, bool extra_pass)
@@ -898,8 +963,25 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
  		/*
  		 * BPF_STX ATOMIC (atomic ops)
  		 */
+		case BPF_STX | BPF_ATOMIC | BPF_B:
+		case BPF_STX | BPF_ATOMIC | BPF_H:
  		case BPF_STX | BPF_ATOMIC | BPF_W:
  		case BPF_STX | BPF_ATOMIC | BPF_DW:
+			if (bpf_atomic_is_load_store(&insn[i])) {
+				ret = emit_atomic_ld_st(insn[i], ctx, image);
+				if (ret)
+					return ret;
+
+				if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
+					addrs[++i] = ctx->idx * 4;
+				break;
+			} else if (size == BPF_B || size == BPF_H) {
+				pr_err_ratelimited(
+					"eBPF filter atomic op code %02x (@%d) unsupported\n",
+					code, i);
+				return -EOPNOTSUPP;
+			}
+
  			save_reg = tmp2_reg;
  			ret_reg = src_reg;
 
tools/testing/selftests/bpf/progs/bpf_misc.h +2 -1
index 530752ddde8e4..c1cfd297aabf1 100644
@@ -229,7 +229,8 @@
 
  #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) &&		\
  	(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) ||	\
-	 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
+	 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) || \
+	  (defined(__TARGET_ARCH_powerpc))
  #define CAN_USE_LOAD_ACQ_STORE_REL
  #endif