Skip to contentSkip to conversation

bpf, arm64: Optimize cast_user code generation

PATCH Superseded Puranjay Mohan 3 participants 3 messages Jul 17 – Jul 18

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Puranjay Mohan

author maintainer
#0 lore raw

PATCH 1/1 bpf, arm64: Optimize cast_user code generation

cast_user converts an arena offset into a user address by combining the low 32 bits of the pointer with the upper 32 bits of user_vm_start, while keeping a NULL pointer NULL. The current sequence always emits six
instructions: it materializes user_vm_start >> 32 into a register, shifts it into place, and ORs in the offset.

The upper half of user_vm_start is a constant, so it can be written directly onto the offset with MOVK. Move the 32-bit offset into dst (which also zeroes the upper 32 bits), then MOVK the non-zero halfwords of the upper address, branching over the MOVKs when the offset is zero so NULL is preserved.

This emits at most four instructions, and only one when the upper half of user_vm_start is zero. The generated code is equivalent.

Before:
; bpf_addr_space_cast(page1, 1, 0);

  7c:   mov     w10, w8
  80:   mov     w8, #1
  84:   lsl     x8, x8, #32
  88:   cbz     x10, 0xffff800087b80c20
  8c:   orr     x10, x8, x10
  90:   mov     x8, x10

After:
; bpf_addr_space_cast(page1, 1, 0);

  7c:   mov     w8, w8
  80:   cbz     w8, 0xffff800087b80c28
  84:   movk    x8, #1, lsl #32
---
 arch/arm64/net/bpf_jit_comp.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)
arch/arm64/net/bpf_jit_comp.c +19 -6
index b0075ece4a6e7..c6756b964c843 100644
@@ -1284,12 +1284,25 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_ALU | BPF_MOV | BPF_X:
 	case BPF_ALU64 | BPF_MOV | BPF_X:
 		if (insn_is_cast_user(insn)) {
-			emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits
-			emit_a64_mov_i(0, dst, ctx->user_vm_start >> 32, ctx);
-			emit(A64_LSL(1, dst, dst, 32), ctx);
-			emit(A64_CBZ(1, tmp, 2), ctx);
-			emit(A64_ORR(1, tmp, dst, tmp), ctx);
-			emit(A64_MOV(1, dst, tmp), ctx);
+			u32 upper = ctx->user_vm_start >> 32;
+			u16 upper_32 = upper & 0xffff;
+			u16 upper_48 = upper >> 16;
+			int nr_movk = !!upper_32 + !!upper_48;
+
+			/*
+			 * Build the user address: the low 32 bits are the arena
+			 * offset, the upper 32 bits come from user_vm_start. A
+			 * zero offset must stay NULL, so branch over the MOVKs
+			 * when it is zero.
+			 */
+			emit(A64_MOV(0, dst, src), ctx); /* 32-bit mov clears the upper 32 bits */
+			if (nr_movk) {
+				emit(A64_CBZ(0, dst, nr_movk + 1), ctx);
+				if (upper_32)
+					emit(A64_MOVK(1, dst, upper_32, 32), ctx);
+				if (upper_48)
+					emit(A64_MOVK(1, dst, upper_48, 48), ctx);
+			}
 			break;
 		} else if (insn_is_mov_percpu_addr(insn)) {
 			if (dst != src)

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d

Signature · 1 line
2.53.0-Meta
#1 lore raw

+cc Xu

Quoted 77 lines · Puranjay Mohan

cast_user converts an arena offset into a user address by combining the low 32 bits of the pointer with the upper 32 bits of user_vm_start, while keeping a NULL pointer NULL. The current sequence always emits six
instructions: it materializes user_vm_start >> 32 into a register, shifts it into place, and ORs in the offset.

The upper half of user_vm_start is a constant, so it can be written directly onto the offset with MOVK. Move the 32-bit offset into dst (which also zeroes the upper 32 bits), then MOVK the non-zero halfwords of the upper address, branching over the MOVKs when the offset is zero so NULL is preserved.

This emits at most four instructions, and only one when the upper half of user_vm_start is zero. The generated code is equivalent.

Before:
; bpf_addr_space_cast(page1, 1, 0);

  7c:   mov     w10, w8
  80:   mov     w8, #1
  84:   lsl     x8, x8, #32
  88:   cbz     x10, 0xffff800087b80c20
  8c:   orr     x10, x8, x10
  90:   mov     x8, x10

After:
; bpf_addr_space_cast(page1, 1, 0);

  7c:   mov     w8, w8
  80:   cbz     w8, 0xffff800087b80c28
  84:   movk    x8, #1, lsl #32
---
 arch/arm64/net/bpf_jit_comp.c | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)
arch/arm64/net/bpf_jit_comp.c +19 -6
index b0075ece4a6e7..c6756b964c843 100644
@@ -1284,12 +1284,25 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
        case BPF_ALU | BPF_MOV | BPF_X:
        case BPF_ALU64 | BPF_MOV | BPF_X:
                if (insn_is_cast_user(insn)) {
-                       emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits
-                       emit_a64_mov_i(0, dst, ctx->user_vm_start >> 32, ctx);
-                       emit(A64_LSL(1, dst, dst, 32), ctx);
-                       emit(A64_CBZ(1, tmp, 2), ctx);
-                       emit(A64_ORR(1, tmp, dst, tmp), ctx);
-                       emit(A64_MOV(1, dst, tmp), ctx);
+                       u32 upper = ctx->user_vm_start >> 32;
+                       u16 upper_32 = upper & 0xffff;
+                       u16 upper_48 = upper >> 16;
+                       int nr_movk = !!upper_32 + !!upper_48;
+
+                       /*
+                        * Build the user address: the low 32 bits are the arena
+                        * offset, the upper 32 bits come from user_vm_start. A
+                        * zero offset must stay NULL, so branch over the MOVKs
+                        * when it is zero.
+                        */
+                       emit(A64_MOV(0, dst, src), ctx); /* 32-bit mov clears the upper 32 bits */
+                       if (nr_movk) {
+                               emit(A64_CBZ(0, dst, nr_movk + 1), ctx);
+                               if (upper_32)
+                                       emit(A64_MOVK(1, dst, upper_32, 32), ctx);
+                               if (upper_48)
+                                       emit(A64_MOVK(1, dst, upper_48, 48), ctx);
+                       }
                        break;
                } else if (insn_is_mov_percpu_addr(insn)) {
                        if (dst != src)

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d --
2.53.0-Meta

reviewer
#2 lore raw
Quoted 68 lines · Puranjay Mohan

+cc Xu

Quoted 65 lines · Puranjay Mohan

cast_user converts an arena offset into a user address by combining the low 32 bits of the pointer with the upper 32 bits of user_vm_start, while keeping a NULL pointer NULL. The current sequence always emits six
instructions: it materializes user_vm_start >> 32 into a register, shifts it into place, and ORs in the offset.

The upper half of user_vm_start is a constant, so it can be written directly onto the offset with MOVK. Move the 32-bit offset into dst (which also zeroes the upper 32 bits), then MOVK the non-zero halfwords of the upper address, branching over the MOVKs when the offset is zero so NULL is preserved.

This emits at most four instructions, and only one when the upper half of user_vm_start is zero. The generated code is equivalent.

Before:

   ; bpf_addr_space_cast(page1, 1, 0);
   7c:   mov     w10, w8
   80:   mov     w8, #1
   84:   lsl     x8, x8, #32
   88:   cbz     x10, 0xffff800087b80c20
   8c:   orr     x10, x8, x10
   90:   mov     x8, x10

After:

   ; bpf_addr_space_cast(page1, 1, 0);
   7c:   mov     w8, w8
   80:   cbz     w8, 0xffff800087b80c28
   84:   movk    x8, #1, lsl #32
---
  arch/arm64/net/bpf_jit_comp.c | 25 +++++++++++++++++++------
  1 file changed, 19 insertions(+), 6 deletions(-)
arch/arm64/net/bpf_jit_comp.c +15 -6
index b0075ece4a6e7..c6756b964c843 100644
@@ -1284,12 +1284,25 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
         case BPF_ALU | BPF_MOV | BPF_X:
         case BPF_ALU64 | BPF_MOV | BPF_X:
                 if (insn_is_cast_user(insn)) {
-                       emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits
-                       emit_a64_mov_i(0, dst, ctx->user_vm_start >> 32, ctx);
-                       emit(A64_LSL(1, dst, dst, 32), ctx);
-                       emit(A64_CBZ(1, tmp, 2), ctx);
-                       emit(A64_ORR(1, tmp, dst, tmp), ctx);
-                       emit(A64_MOV(1, dst, tmp), ctx);
+                       u32 upper = ctx->user_vm_start >> 32;
+                       u16 upper_32 = upper & 0xffff;
+                       u16 upper_48 = upper >> 16;
+                       int nr_movk = !!upper_32 + !!upper_48;
+
+                       /*
+                        * Build the user address: the low 32 bits are the arena
+                        * offset, the upper 32 bits come from user_vm_start. A
+                        * zero offset must stay NULL, so branch over the MOVKs
+                        * when it is zero.
+                        */
+                       emit(A64_MOV(0, dst, src), ctx); /* 32-bit mov clears the upper 32 bits */
+                       if (nr_movk) {
+                               emit(A64_CBZ(0, dst, nr_movk + 1), ctx);
+                               if (upper_32)
Quoted 8 lines

arch/arm64/net/bpf_jit_comp.c static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn · line 1308

Quoted 8 lines
arch/arm64/net/bpf_jit_comp.c +4 -0
+                                       emit(A64_MOVK(1, dst, upper_32, 32), ctx);
+                               if (upper_48)
+                                       emit(A64_MOVK(1, dst, upper_48, 48), ctx);
+                       }
                         break;
                 } else if (insn_is_mov_percpu_addr(insn)) {
                         if (dst != src)

The upper_32/48 names sound like the upper 32/48 bits from the value, how about change to upper_lo and upper_high?

Otherwise looks good to me.

Quoted 4 lines
Quoted 4 lines

base-commit: 1d91ea01185656ac3ee63c5f9f6f8bde3c746b3d --
2.53.0-Meta