Skip to contentSkip to conversation

Improve verifier log readability

PATCH Superseded Christy Lee 3 participants 10 messages Dec 15, 2021 – Dec 16, 2021 2 messages not in the archive

Subsystems (4)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover Improve verifier log readability

Simplify verifier logs and improve readability.

Changelog:
----------
v1 -> v2:
v1: https://lore.kernel.org/bpf/[email protected]/

Patch 2/3:
* Verifier will skip insn_idx when the insn is longer than 8 bytes, example:
0000000000000000 <good_prog>:

       0:	18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00	r2 = 0 ll
       2:	63 12 00 00 00 00 00 00	*(u32 *)(r2 + 0) = r1
       3:	61 20 04 00 00 00 00 00	r0 = *(u32 *)(r2 + 4)
       4:	95 00 00 00 00 00 00 00	exit

It's incorrect to check that prev_insn_idx = insn_idx - 1, skip this check and print the verifier state on the correct line.
Before:
0: R1=ctx(id=0,off=0,imm=0) R10=fp0
; a[0] = (int)(long)ctx;
0: (18) r2 = 0xffffc900006de000
2: R2_w=map_value(id=0,off=0,ks=4,vs=16,imm=0)
2: (63) *(u32 *)(r2 +0) = r1
After:
0: R1=ctx(id=0,off=0,imm=0) R10=fp0
; a[0] = (int)(long)ctx;
0: (18) r2 = 0xffffc900006de000 ; R2_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) 2: (63) *(u32 *)(r2 +0) = r1
* Track previous line logging length in env, allow aligning intsruction from anywhere in the verifier
* Fixed bug where the verifier printed verifier state after checking source memory access but before check destination memory access, this ensures the aligned verifier state contains all scratched registers

Patch 3/3:
* Added one more case where we should only log in log_level=2

Christy Lee (3):
Only print scratched registers and stack slots to verifier logs Right align verifier states in verifier logs
Only output backtracking information in log level 2

 include/linux/bpf_verifier.h                  |  10 +
 kernel/bpf/verifier.c                         | 131 +++++++++--
 .../testing/selftests/bpf/prog_tests/align.c  | 214 +++++++++---------
 3 files changed, 232 insertions(+), 123 deletions(-)

--
2.30.2

#1 lore raw

PATCH v2 1/3 Only print scratched registers and stack slots to verifier logs

When printing verifier state for any log level, print full verifier state only on function calls or on errors. Otherwise, only print the registers and stack slots that were accessed.

Log size differences:

verif_scale_loop6 before: 234566564
verif_scale_loop6 after: 72143943
69% size reduction

kfree_skb before: 166406
kfree_skb after: 55386
69% size reduction

Before:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=invP0 R10=fp0 fp-8_w=00000000 fp-16_w=00\
000000 fp-24_w=00000000 fp-32_w=00000000 fp-40_w=00000000 fp-48_w=00000000 fp-56_w=00000000 fp-64_w=00000000 fp-72_w=00000000 fp-80_w=00000\
000 fp-88_w=00000000 fp-96_w=00000000 fp-104_w=00000000 fp-112_w=00000000 fp-120_w=00000000 fp-128_w=00000000 fp-136_w=00000000 fp-144_w=00\
000000 fp-152_w=00000000 fp-160_w=00000000 fp-168_w=00000000 fp-176_w=00000000 fp-184_w=00000000 fp-192_w=00000000 fp-200_w=00000000 fp-208\
_w=00000000 fp-216_w=00000000 fp-224_w=00000000 fp-232_w=00000000 fp-240_w=00000000 fp-248_w=00000000 fp-256_w=00000000 fp-264_w=00000000 f\
p-272_w=00000000 fp-280_w=00000000 fp-288_w=00000000 fp-296_w=00000000 fp-304_w=00000000 fp-312_w=00000000 fp-320_w=00000000 fp-328_w=00000\
000 fp-336_w=00000000 fp-344_w=00000000 fp-352_w=00000000 fp-360_w=00000000 fp-368_w=00000000 fp-376_w=00000000 fp-384_w=00000000 fp-392_w=\
00000000 fp-400_w=00000000 fp-408_w=00000000 fp-416_w=00000000 fp-424_w=00000000 fp-432_w=00000000 fp-440_w=00000000 fp-448_w=00000000 ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1) R10=fp0
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=1) R10=fp0
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0

After:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1)
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R3=invP(id=0)

---
 include/linux/bpf_verifier.h                  |  7 +++
 kernel/bpf/verifier.c                         | 62 ++++++++++++++++++-
 .../testing/selftests/bpf/prog_tests/align.c  | 30 ++++-----
 3 files changed, 82 insertions(+), 17 deletions(-)
include/linux/bpf_verifier.h +7 -0
index 182b16a91084..c555222c97d6 100644
@@ -474,6 +474,13 @@ struct bpf_verifier_env {
 	/* longest register parentage chain walked for liveness marking */
 	u32 longest_mark_read_walk;
 	bpfptr_t fd_array;
+
+	/* bit mask to keep track of whether a register has been accessed
+	 * since the last time the function state was pritned
+	 */
+	u32 scratched_regs;
+	/* Same as scratched_regs but for stack slots */
+	u64 scratched_stack_slots;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
kernel/bpf/verifier.c +60 -2

156 lines not on this page · raw message

tools/testing/selftests/bpf/prog_tests/align.c +15 -15
index 837f67c6bfda..aeb2080a67f7 100644
@@ -39,8 +39,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv2"},
 			{2, "R3_w=inv4"},
 			{3, "R3_w=inv8"},
@@ -67,8 +67,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv1"},
 			{2, "R3_w=inv2"},
 			{3, "R3_w=inv4"},
@@ -96,8 +96,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv4"},
 			{2, "R3_w=inv8"},
 			{3, "R3_w=inv10"},
@@ -118,8 +118,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv7"},
 			{2, "R3_w=inv7"},
 			{3, "R3_w=inv14"},
@@ -161,13 +161,13 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
+			{6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
 			{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
 			{9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			{10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
 			{11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
-			{18, "R3=pkt_end(id=0,off=0,imm=0)"},
+			{13, "R3_w=pkt_end(id=0,off=0,imm=0)"},
 			{18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
 			{20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
@@ -234,10 +234,10 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
+			{3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
 			{5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
 			{6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
-			{10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
+			{9, "R2=pkt(id=0,off=0,r=18,imm=0)"},
 			{10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
 			{10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
@@ -296,7 +296,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+			{6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
 			{8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			/* Offset is added to packet pointer R5, resulting in
 			 * known fixed offset, and variable offset from R6.
@@ -386,7 +386,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+			{6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
 			{8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			/* Adding 14 makes R6 be (4n+2) */
 			{9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
@@ -458,7 +458,7 @@ static struct bpf_align_test tests[] = {
 			/* Checked s>=0 */
 			{9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			/* packet pointer + nonnegative (4n+2) */
-			{11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
+			{12, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			{13, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			/* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine.
 			 * We checked the bounds, but it might have been able
Signature · 1 line
2.30.2
Acked-by on v2 1/3
#2 lore raw

Re: [PATCH v2 bpf-next 1/3] Only print scratched registers and stack slots to verifier logs

Quoted 70 lines · Christy Lee

When printing verifier state for any log level, print full verifier state only on function calls or on errors. Otherwise, only print the registers and stack slots that were accessed.

Log size differences:

verif_scale_loop6 before: 234566564
verif_scale_loop6 after: 72143943
69% size reduction

kfree_skb before: 166406
kfree_skb after: 55386
69% size reduction

Before:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=invP0 R10=fp0 fp-8_w=00000000 fp-16_w=00\
000000 fp-24_w=00000000 fp-32_w=00000000 fp-40_w=00000000 fp-48_w=00000000 fp-56_w=00000000 fp-64_w=00000000 fp-72_w=00000000 fp-80_w=00000\
000 fp-88_w=00000000 fp-96_w=00000000 fp-104_w=00000000 fp-112_w=00000000 fp-120_w=00000000 fp-128_w=00000000 fp-136_w=00000000 fp-144_w=00\
000000 fp-152_w=00000000 fp-160_w=00000000 fp-168_w=00000000 fp-176_w=00000000 fp-184_w=00000000 fp-192_w=00000000 fp-200_w=00000000 fp-208\
_w=00000000 fp-216_w=00000000 fp-224_w=00000000 fp-232_w=00000000 fp-240_w=00000000 fp-248_w=00000000 fp-256_w=00000000 fp-264_w=00000000 f\
p-272_w=00000000 fp-280_w=00000000 fp-288_w=00000000 fp-296_w=00000000 fp-304_w=00000000 fp-312_w=00000000 fp-320_w=00000000 fp-328_w=00000\
000 fp-336_w=00000000 fp-344_w=00000000 fp-352_w=00000000 fp-360_w=00000000 fp-368_w=00000000 fp-376_w=00000000 fp-384_w=00000000 fp-392_w=\
00000000 fp-400_w=00000000 fp-408_w=00000000 fp-416_w=00000000 fp-424_w=00000000 fp-432_w=00000000 fp-440_w=00000000 fp-448_w=00000000 ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1) R10=fp0
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=1) R10=fp0
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0

After:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1)
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R3=invP(id=0)

Quoted 2 lines

---

Looks good to me, but see a few nits below. They can be ignored or addressed as a follow up.

Quoted 10 lines
 include/linux/bpf_verifier.h                  |  7 +++
 kernel/bpf/verifier.c                         | 62 ++++++++++++++++++-
 .../testing/selftests/bpf/prog_tests/align.c  | 30 ++++-----
 3 files changed, 82 insertions(+), 17 deletions(-)
include/linux/bpf_verifier.h +0 -0
index 182b16a91084..c555222c97d6 100644

include/linux/bpf_verifier.h struct bpf_verifier_env { · line 479

Quoted 6 lines
include/linux/bpf_verifier.h +3 -0
@@ -474,6 +474,13 @@ struct bpf_verifier_env {
        /* longest register parentage chain walked for liveness marking */
        u32 longest_mark_read_walk;
        bpfptr_t fd_array;
+
+       /* bit mask to keep track of whether a register has been accessed
+        * since the last time the function state was pritned

typo: printed

Quoted 7 lines
include/linux/bpf_verifier.h +4 -0
+        */
+       u32 scratched_regs;
+       /* Same as scratched_regs but for stack slots */
+       u64 scratched_stack_slots;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,

[...]

Quoted 13 lines
include/linux/bpf_verifier.h +2 -0
+       mark_stack_slot_scratched(env, spi);
        if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
            !register_is_null(reg) && env->bpf_capable) {
                if (dst_reg != BPF_REG_FP) {
@@ -2957,6 +3004,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
                slot = -i - 1;
                spi = slot / BPF_REG_SIZE;
                stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
+               mark_stack_slot_scratched(env, spi);
 
                if (!env->allow_ptr_leaks
                                && *stype != NOT_INIT

include/linux/bpf_verifier.h static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn · line 6062

Quoted 6 lines
include/linux/bpf_verifier.h +1 -0
@@ -6009,8 +6057,10 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
        *insn_idx = env->subprog_info[subprog].start - 1;
 
        if (env->log.level & BPF_LOG_LEVEL) {
+               mark_verifier_state_scratched(env);
                verbose(env, "caller:\n");
                print_verifier_state(env, caller);

In all but one cases you call mark_verifier_state_scratched(env) before calling print_verifier_state(). Instead of sort of artificially scratch entire state, I'd add a bool flag to print_verifier_state() to control whether we want to take into account scratch masks or not. It would also make sure that we will never forget to scratch it where necessary (you'll have to make a conscious decision on each print_verifier_state() call).

Quoted 4 lines
include/linux/bpf_verifier.h +1 -0
+               mark_verifier_state_scratched(env);
                verbose(env, "callee:\n");
                print_verifier_state(env, callee);
        }

[...]

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 169

Quoted 11 lines
include/linux/bpf_verifier.h +1 -1
@@ -161,13 +161,13 @@ static struct bpf_align_test tests[] = {
                },
                .prog_type = BPF_PROG_TYPE_SCHED_CLS,
                .matches = {
-                       {7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
+                       {6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
                        {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
                        {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
                        {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
                        {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},

consider this a feature request (unless people disagree), but these "_value" suffixes for umin/umax/s32_min/etc are just noise and don't contribute anything but extra visual noise. I'd remove them. map_value is ok, probably (because we should have map_key, I guess).

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 237

Quoted 11 lines
include/linux/bpf_verifier.h +2 -2
-                       {18, "R3=pkt_end(id=0,off=0,imm=0)"},
+                       {13, "R3_w=pkt_end(id=0,off=0,imm=0)"},
                        {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
                        {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
@@ -234,10 +234,10 @@ static struct bpf_align_test tests[] = {
                },
                .prog_type = BPF_PROG_TYPE_SCHED_CLS,
                .matches = {
-                       {4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
+                       {3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},

another improvement suggestion: all these id=0 is also noise. I'd make sure that cases where we do care about ids always use id > 0 (I think that might be the case already) and just never output id=0

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 243

Quoted 7 lines
include/linux/bpf_verifier.h +1 -1
                        {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
                        {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
-                       {10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
+                       {9, "R2=pkt(id=0,off=0,r=18,imm=0)"},
                        {10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
                        {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},

Another thing that always confuses me is the use of "inv" to denote scalar. I could never understand why it's not a "scalar" or just nothing. Especially for cases when we have known value. We'll see "R4=inv4". So confusing. Maybe let's just drop the inv, so we'll have:

R4=4

or (taking into the account all the above suggestions)

R4=(umax=65535,var_off=(0x0; 0xffff))

I don't think we lose anything by dropping "inv", but "scalar" would be still better (even if slightly longer).

Quoted 9 lines
include/linux/bpf_verifier.h +1 -1
@@ -296,7 +296,7 @@ static struct bpf_align_test tests[] = {
                        /* Calculated offset in R6 has unknown value, but known
                         * alignment of 4.
                         */
-                       {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+                       {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
                        {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
                        /* Offset is added to packet pointer R5, resulting in
                         * known fixed offset, and variable offset from R6.

[...]

A message that is not in the archive (sent off-list or not yet received): CAPqJDZpuZ586TawooQhxZGvOqgF4yCjT0yOLEXAUer8eJDO7gw@mail.gmail.com

#4 lore raw

Re: [PATCH v2 bpf-next 1/3] Only print scratched registers and stack slots to verifier logs

On Thu, Dec 16, 2021 at 7:48 AM Christy Lee <[email protected]> wrote:

Apologies, resending this because the previous email was not plain text.

Quoted 152 lines · Andrii Nakryiko
Quoted 3 lines · Christy Lee

[...]

Quoted 52 lines

Looks good to me, but see a few nits below. They can be ignored or addressed as a follow up.

Quoted 10 lines
 include/linux/bpf_verifier.h                  |  7 +++
 kernel/bpf/verifier.c                         | 62 ++++++++++++++++++-
 .../testing/selftests/bpf/prog_tests/align.c  | 30 ++++-----
 3 files changed, 82 insertions(+), 17 deletions(-)
include/linux/bpf_verifier.h +0 -0
index 182b16a91084..c555222c97d6 100644

include/linux/bpf_verifier.h struct bpf_verifier_env { · line 479

Quoted 6 lines
include/linux/bpf_verifier.h +3 -0
@@ -474,6 +474,13 @@ struct bpf_verifier_env {
        /* longest register parentage chain walked for liveness marking */
        u32 longest_mark_read_walk;
        bpfptr_t fd_array;
+
+       /* bit mask to keep track of whether a register has been accessed
+        * since the last time the function state was pritned

typo: printed

Quoted 7 lines
include/linux/bpf_verifier.h +4 -0
+        */
+       u32 scratched_regs;
+       /* Same as scratched_regs but for stack slots */
+       u64 scratched_stack_slots;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,

[...]

Quoted 15 lines
include/linux/bpf_verifier.h +2 -0
+       mark_stack_slot_scratched(env, spi);
        if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
            !register_is_null(reg) && env->bpf_capable) {
                if (dst_reg != BPF_REG_FP) {
@@ -2957,6 +3004,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
                slot = -i - 1;
                spi = slot / BPF_REG_SIZE;
                stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
+               mark_stack_slot_scratched(env, spi);
 
                if (!env->allow_ptr_leaks
                                && *stype != NOT_INIT
@@ -6009,8 +6057,10 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
        *insn_idx = env->subprog_info[subprog].start - 1;
 
Quoted 12 lines

include/linux/bpf_verifier.h static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn · line 6062

Quoted 4 lines
include/linux/bpf_verifier.h +1 -0
        if (env->log.level & BPF_LOG_LEVEL) {
+               mark_verifier_state_scratched(env);
                verbose(env, "caller:\n");
                print_verifier_state(env, caller);

In all but one cases you call mark_verifier_state_scratched(env) before calling print_verifier_state(). Instead of sort of artificially scratch entire state, I'd add a bool flag to print_verifier_state() to control whether we want to take into account scratch masks or not. It would also make sure that we will never forget to scratch it where necessary (you'll have to make a conscious decision on each print_verifier_state() call).

Good idea! I'll do that.

Quoted 9 lines
Quoted 6 lines
include/linux/bpf_verifier.h +1 -0
+               mark_verifier_state_scratched(env);
                verbose(env, "callee:\n");
                print_verifier_state(env, callee);
        }

[...]

Quoted 12 lines

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 169

Quoted 11 lines
include/linux/bpf_verifier.h +1 -1
@@ -161,13 +161,13 @@ static struct bpf_align_test tests[] = {
                },
                .prog_type = BPF_PROG_TYPE_SCHED_CLS,
                .matches = {
-                       {7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
+                       {6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
                        {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
                        {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
                        {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
                        {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},

These would be very useful features, I'll address them as a follow up.

Quoted 57 lines

consider this a feature request (unless people disagree), but these "_value" suffixes for umin/umax/s32_min/etc are just noise and don't contribute anything but extra visual noise. I'd remove them. map_value is ok, probably (because we should have map_key, I guess).

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 237

Quoted 11 lines
include/linux/bpf_verifier.h +2 -2
-                       {18, "R3=pkt_end(id=0,off=0,imm=0)"},
+                       {13, "R3_w=pkt_end(id=0,off=0,imm=0)"},
                        {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
                        {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
@@ -234,10 +234,10 @@ static struct bpf_align_test tests[] = {
                },
                .prog_type = BPF_PROG_TYPE_SCHED_CLS,
                .matches = {
-                       {4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
+                       {3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},

another improvement suggestion: all these id=0 is also noise. I'd make sure that cases where we do care about ids always use id > 0 (I think that might be the case already) and just never output id=0

include/linux/bpf_verifier.h static struct bpf_align_test tests[] = { · line 243

Quoted 7 lines
include/linux/bpf_verifier.h +1 -1
                        {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
                        {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
-                       {10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
+                       {9, "R2=pkt(id=0,off=0,r=18,imm=0)"},
                        {10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
                        {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
                        {14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},

Another thing that always confuses me is the use of "inv" to denote scalar. I could never understand why it's not a "scalar" or just nothing. Especially for cases when we have known value. We'll see "R4=inv4". So confusing. Maybe let's just drop the inv, so we'll have:

R4=4

or (taking into the account all the above suggestions)

R4=(umax=65535,var_off=(0x0; 0xffff))

I don't think we lose anything by dropping "inv", but "scalar" would be still better (even if slightly longer).

Quoted 9 lines
include/linux/bpf_verifier.h +1 -1
@@ -296,7 +296,7 @@ static struct bpf_align_test tests[] = {
                        /* Calculated offset in R6 has unknown value, but known
                         * alignment of 4.
                         */
-                       {8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+                       {6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
                        {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
                        /* Offset is added to packet pointer R5, resulting in
                         * known fixed offset, and variable offset from R6.

[...]

#5 lore raw

PATCH v2 2/3 Right align verifier states in verifier logs

In reply to Christy Lee

Make the verifier logs more readable, print the verifier states on the corresponding instruction line. If the previous line was not a bpf instruction, then print the verifier states on its own line.

Before:

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2
87: R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1
88: R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6
89: R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0
91: R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123
92: R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

After:

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2 ; R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1 ; R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
88: (bf) r1 = r6 ; R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0 ; R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
91: (b7) r1 = 123 ; R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

---
 include/linux/bpf_verifier.h                  |   3 +
 kernel/bpf/verifier.c                         |  69 ++++--
 .../testing/selftests/bpf/prog_tests/align.c  | 196 ++++++++++--------
 3 files changed, 156 insertions(+), 112 deletions(-)
include/linux/bpf_verifier.h +3 -0
index c555222c97d6..cf5801c02216 100644
@@ -388,6 +388,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
 #define BPF_LOG_LEVEL	(BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
 #define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS)
 #define BPF_LOG_KERNEL	(BPF_LOG_MASK + 1) /* kernel internal flag */
+#define BPF_LOG_MIN_ALIGNMENT 8
+#define BPF_LOG_ALIGNMENT_POS 32
 
 static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 {
@@ -481,6 +483,7 @@ struct bpf_verifier_env {
 	u32 scratched_regs;
 	/* Same as scratched_regs but for stack slots */
 	u64 scratched_stack_slots;
+	u32 prev_log_len, prev_insn_print_len;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
kernel/bpf/verifier.c +49 -20

122 lines not on this page · raw message

tools/testing/selftests/bpf/prog_tests/align.c +104 -92

368 lines not on this page · raw message

Signature · 1 line
2.30.2
#6 lore raw

Re: [PATCH v2 bpf-next 2/3] Right align verifier states in verifier logs

Quoted 32 lines · Christy Lee

Make the verifier logs more readable, print the verifier states on the corresponding instruction line. If the previous line was not a bpf instruction, then print the verifier states on its own line.

Before:

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2
87: R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1
88: R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6
89: R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0
91: R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123
92: R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

After:

Quoted 16 lines

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2 ; R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1 ; R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
88: (bf) r1 = r6 ; R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0 ; R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
91: (b7) r1 = 123 ; R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

There seems to be quite a lot of jumpin back and forth in terms of 33th (see off by one error below) vs 40th offsets (this is for pyperf50 test):

16: (07) r2 += -8 ; R2_w=fp-8
; Event* event = bpf_map_lookup_elem(&eventmap, &zero);
17: (18) r1 = 0xffff88810d81dc00 ;
R1_w=map_ptr(id=0,off=0,ks=4,vs=252,imm=0)
19: (85) call bpf_map_lookup_elem#1 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
20: (bf) r7 = r0 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0) R7_w=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
; if (!event)
21: (15) if r7 == 0x0 goto pc+5193 ;
R7_w=map_value(id=0,off=0,ks=4,vs=252,imm=0)
; event->pid = pid;
22: (61) r1 = *(u32 *)(r10 -4) ; R1_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0

Maybe let's bump the minimum to 40?

Quoted 34 lines
---
 include/linux/bpf_verifier.h                  |   3 +
 kernel/bpf/verifier.c                         |  69 ++++--
 .../testing/selftests/bpf/prog_tests/align.c  | 196 ++++++++++--------
 3 files changed, 156 insertions(+), 112 deletions(-)
include/linux/bpf_verifier.h +3 -0
index c555222c97d6..cf5801c02216 100644
@@ -388,6 +388,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
 #define BPF_LOG_LEVEL  (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
 #define BPF_LOG_MASK   (BPF_LOG_LEVEL | BPF_LOG_STATS)
 #define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
+#define BPF_LOG_MIN_ALIGNMENT 8
+#define BPF_LOG_ALIGNMENT_POS 32
 
 static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 {
@@ -481,6 +483,7 @@ struct bpf_verifier_env {
        u32 scratched_regs;
        /* Same as scratched_regs but for stack slots */
        u64 scratched_stack_slots;
+       u32 prev_log_len, prev_insn_print_len;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
kernel/bpf/verifier.c +0 -0
index f4228864a3e9..a8f1426b0367 100644

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 801

Quoted 7 lines
kernel/bpf/verifier.c +4 -0
@@ -795,6 +795,27 @@ static void print_verifier_state(struct bpf_verifier_env *env,
        mark_verifier_state_clean(env);
 }
 
+static u32 vlog_alignment(u32 prev_insn_print_len)
+{
+       if (prev_insn_print_len < BPF_LOG_ALIGNMENT_POS)
+               return BPF_LOG_ALIGNMENT_POS - prev_insn_print_len + 1;

why +1 here?

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 811

Quoted 10 lines
kernel/bpf/verifier.c +10 -0
+       return round_up(prev_insn_print_len, BPF_LOG_MIN_ALIGNMENT) -
+              prev_insn_print_len;
+}
+
+static void print_prev_insn_state(struct bpf_verifier_env *env,
+                                 const struct bpf_func_state *state)
+{
+       if (env->prev_log_len == env->log.len_used) {
+               if (env->prev_log_len)
+                       bpf_vlog_reset(&env->log, env->prev_log_len - 1);

I don't get this... why do we need this reset? Why just appending alignment spaces below doesn't work?

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 813

Quoted 2 lines
kernel/bpf/verifier.c +2 -0
+               verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len),
+                       ' ');

nit: keep it on single line

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 815

Quoted 2 lines
kernel/bpf/verifier.c +2 -0
+       } else
+               verbose(env, "%d:", env->insn_idx);

if one branch of if/else has {}, the other one has to have them as well, even if it's a single line statement

Quoted 6 lines
kernel/bpf/verifier.c +3 -0
+       print_verifier_state(env, state);
+}
+
 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  * small to hold src. This is different from krealloc since we don't want to preserve
  * the contents of dst.

[...]

kernel/bpf/verifier.c static int check_cond_jmp_op(struct bpf_verifier_env *env, · line 9470

Quoted 9 lines
kernel/bpf/verifier.c +3 -2
@@ -9441,8 +9465,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
                        insn->dst_reg);
                return -EACCES;
        }
-       if (env->log.level & BPF_LOG_LEVEL)
-               print_verifier_state(env, this_branch->frame[this_branch->curframe]);
+       if (env->log.level & BPF_LOG_LEVEL) {
+               print_prev_insn_state(
+                       env, this_branch->frame[this_branch->curframe]);

nit: keep on a single line. But also is it really a "previous instruction" or still a current instruction? Maybe just "print_insn_state"? Do we have "next_insn" helper anywhere? If not, dropping this "prev_" prefix from helpers and variables would be cleaner, IMO

Quoted 10 lines
kernel/bpf/verifier.c +1 -2
+       }
        return 0;
 }
 
@@ -11310,17 +11336,12 @@ static int do_check(struct bpf_verifier_env *env)
                if (need_resched())
                        cond_resched();
 
-               if ((env->log.level & BPF_LOG_LEVEL2) ||
-                   (env->log.level & BPF_LOG_LEVEL && do_print_state)) {

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11339

Quoted 10 lines
kernel/bpf/verifier.c +1 -9
-                       if (verifier_state_scratched(env) &&
-                           (env->log.level & BPF_LOG_LEVEL2))
-                               verbose(env, "%d:", env->insn_idx);
-                       else
-                               verbose(env, "\nfrom %d to %d%s:",
-                                       env->prev_insn_idx, env->insn_idx,
-                                       env->cur_state->speculative ?
-                                       " (speculative execution)" : "");
-                       print_verifier_state(env, state->frame[state->curframe]);
+               if (env->log.level & BPF_LOG_LEVEL1 && do_print_state) {

() around bit operations

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11344

Quoted 5 lines
kernel/bpf/verifier.c +5 -0
+                       verbose(env, "\nfrom %d to %d%s:\n", env->prev_insn_idx,
+                               env->insn_idx,
+                               env->cur_state->speculative ?
+                                       " (speculative execution)" :
+                                             "");

it's ok to go up to 100 characters, please keep the code more readable

Quoted 21 lines
kernel/bpf/verifier.c +8 -0
                        do_print_state = false;
                }
 
@@ -11331,9 +11352,17 @@ static int do_check(struct bpf_verifier_env *env)
                                .private_data   = env,
                        };
 
+                       if (verifier_state_scratched(env))
+                               print_prev_insn_state(
+                                       env, state->frame[state->curframe]);
+
                        verbose_linfo(env, env->insn_idx, "; ");
+                       env->prev_log_len = env->log.len_used;
                        verbose(env, "%d: ", env->insn_idx);
                        print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
+                       env->prev_insn_print_len =
+                               env->log.len_used - env->prev_log_len;
+                       env->prev_log_len = env->log.len_used;
                }
 
                if (bpf_prog_is_dev_bound(env->prog->aux)) {

[...]

A message that is not in the archive (sent off-list or not yet received): CAPqJDZr1AudE2PfbZQarHf0N5i_-xm-zyhfLqS5rogX98UtNLQ@mail.gmail.com

#8 lore raw

Re: [PATCH v2 bpf-next 2/3] Right align verifier states in verifier logs

On Thu, Dec 16, 2021 at 7:59 AM Christy Lee <[email protected]> wrote:>

Apologies, resending this because the previous email was not plain text.

Quoted 164 lines · Andrii Nakryiko
Quoted 3 lines · Christy Lee
Quoted 1 line

[...]

Quoted 22 lines

There seems to be quite a lot of jumpin back and forth in terms of 33th (see off by one error below) vs 40th offsets (this is for pyperf50 test):

16: (07) r2 += -8 ; R2_w=fp-8
; Event* event = bpf_map_lookup_elem(&eventmap, &zero);
17: (18) r1 = 0xffff88810d81dc00 ;
R1_w=map_ptr(id=0,off=0,ks=4,vs=252,imm=0)
19: (85) call bpf_map_lookup_elem#1 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
20: (bf) r7 = r0 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0) R7_w=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
; if (!event)
21: (15) if r7 == 0x0 goto pc+5193 ;
R7_w=map_value(id=0,off=0,ks=4,vs=252,imm=0)
; event->pid = pid;
22: (61) r1 = *(u32 *)(r10 -4) ; R1_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0

Quoted 1 line

Maybe let's bump the minimum to 40?

Will do! I'll experiment with the nicest looking offset.

Quoted 4 lines
Quoted 4 lines

[...]

Quoted 8 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 801

Quoted 6 lines
kernel/bpf/verifier.c +4 -0
+static u32 vlog_alignment(u32 prev_insn_print_len)
+{
+       if (prev_insn_print_len < BPF_LOG_ALIGNMENT_POS)
+               return BPF_LOG_ALIGNMENT_POS - prev_insn_print_len + 1;

why +1 here?

I wanted the insn_state to be prepended by BPF_LOG_ALIGNMENT_POS number of characters, in this case, that would be prepended by 32 and starting at 33. This ensures that whether prev_insn_print_len is smaller than BPF_LOG_ALIGNMENT_POS or not, the insn_states would be aligned properly, there would be an off-by-one error otherwise.

Quoted 15 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 811

Quoted 12 lines
kernel/bpf/verifier.c +10 -0
+       return round_up(prev_insn_print_len, BPF_LOG_MIN_ALIGNMENT) -
+              prev_insn_print_len;
+}
+
+static void print_prev_insn_state(struct bpf_verifier_env *env,
+                                 const struct bpf_func_state *state)
+{
+       if (env->prev_log_len == env->log.len_used) {
+               if (env->prev_log_len)
+                       bpf_vlog_reset(&env->log, env->prev_log_len - 1);

I don't get this... why do we need this reset? Why just appending alignment spaces below doesn't work?

The insn are printed via print_bpf_insn() which ends the line with a '\n', we need to reset one character in order to remove the new line and print insn_state on the same line. print_bpf_insn() is defined in kernel/bpf/disasm.h and used by bpf_tool as well, so I didn't want to modify it.

Quoted 92 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 813

Quoted 4 lines
kernel/bpf/verifier.c +2 -0
+               verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len),
+                       ' ');

nit: keep it on single line

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 815

Quoted 2 lines
kernel/bpf/verifier.c +2 -0
+       } else
+               verbose(env, "%d:", env->insn_idx);

if one branch of if/else has {}, the other one has to have them as well, even if it's a single line statement

Quoted 6 lines
kernel/bpf/verifier.c +3 -0
+       print_verifier_state(env, state);
+}
+
 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  * small to hold src. This is different from krealloc since we don't want to preserve
  * the contents of dst.

[...]

kernel/bpf/verifier.c static int check_cond_jmp_op(struct bpf_verifier_env *env, · line 9470

Quoted 9 lines
kernel/bpf/verifier.c +3 -2
@@ -9441,8 +9465,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
                        insn->dst_reg);
                return -EACCES;
        }
-       if (env->log.level & BPF_LOG_LEVEL)
-               print_verifier_state(env, this_branch->frame[this_branch->curframe]);
+       if (env->log.level & BPF_LOG_LEVEL) {
+               print_prev_insn_state(
+                       env, this_branch->frame[this_branch->curframe]);

nit: keep on a single line. But also is it really a "previous instruction" or still a current instruction? Maybe just "print_insn_state"? Do we have "next_insn" helper anywhere? If not, dropping this "prev_" prefix from helpers and variables would be cleaner, IMO

Quoted 10 lines
kernel/bpf/verifier.c +1 -2
+       }
        return 0;
 }
 
@@ -11310,17 +11336,12 @@ static int do_check(struct bpf_verifier_env *env)
                if (need_resched())
                        cond_resched();
 
-               if ((env->log.level & BPF_LOG_LEVEL2) ||
-                   (env->log.level & BPF_LOG_LEVEL && do_print_state)) {

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11339

Quoted 10 lines
kernel/bpf/verifier.c +1 -9
-                       if (verifier_state_scratched(env) &&
-                           (env->log.level & BPF_LOG_LEVEL2))
-                               verbose(env, "%d:", env->insn_idx);
-                       else
-                               verbose(env, "\nfrom %d to %d%s:",
-                                       env->prev_insn_idx, env->insn_idx,
-                                       env->cur_state->speculative ?
-                                       " (speculative execution)" : "");
-                       print_verifier_state(env, state->frame[state->curframe]);
+               if (env->log.level & BPF_LOG_LEVEL1 && do_print_state) {

() around bit operations

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11344

Quoted 5 lines
kernel/bpf/verifier.c +5 -0
+                       verbose(env, "\nfrom %d to %d%s:\n", env->prev_insn_idx,
+                               env->insn_idx,
+                               env->cur_state->speculative ?
+                                       " (speculative execution)" :
+                                             "");

it's ok to go up to 100 characters, please keep the code more readable

Quoted 21 lines
kernel/bpf/verifier.c +8 -0
                        do_print_state = false;
                }
 
@@ -11331,9 +11352,17 @@ static int do_check(struct bpf_verifier_env *env)
                                .private_data   = env,
                        };
 
+                       if (verifier_state_scratched(env))
+                               print_prev_insn_state(
+                                       env, state->frame[state->curframe]);
+
                        verbose_linfo(env, env->insn_idx, "; ");
+                       env->prev_log_len = env->log.len_used;
                        verbose(env, "%d: ", env->insn_idx);
                        print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
+                       env->prev_insn_print_len =
+                               env->log.len_used - env->prev_log_len;
+                       env->prev_log_len = env->log.len_used;
                }
 
                if (bpf_prog_is_dev_bound(env->prog->aux)) {

[...]

#9 lore raw

Re: [PATCH v2 bpf-next 2/3] Right align verifier states in verifier logs

Quoted 77 lines · Christy Lee

On Thu, Dec 16, 2021 at 7:59 AM Christy Lee <[email protected]> wrote:>

Apologies, resending this because the previous email was not plain text.

Quoted 72 lines · Andrii Nakryiko
Quoted 3 lines · Christy Lee
Quoted 1 line

[...]

Quoted 22 lines

There seems to be quite a lot of jumpin back and forth in terms of 33th (see off by one error below) vs 40th offsets (this is for pyperf50 test):

16: (07) r2 += -8 ; R2_w=fp-8
; Event* event = bpf_map_lookup_elem(&eventmap, &zero);
17: (18) r1 = 0xffff88810d81dc00 ;
R1_w=map_ptr(id=0,off=0,ks=4,vs=252,imm=0)
19: (85) call bpf_map_lookup_elem#1 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
20: (bf) r7 = r0 ;
R0=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0) R7_w=map_value_or_null(id=3,off=0,ks=4,vs=252,imm=0)
; if (!event)
21: (15) if r7 == 0x0 goto pc+5193 ;
R7_w=map_value(id=0,off=0,ks=4,vs=252,imm=0)
; event->pid = pid;
22: (61) r1 = *(u32 *)(r10 -4) ; R1_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0

Quoted 1 line

Maybe let's bump the minimum to 40?

Will do! I'll experiment with the nicest looking offset.

Quoted 4 lines
Quoted 4 lines

[...]

Quoted 8 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 801

Quoted 6 lines
kernel/bpf/verifier.c +4 -0
+static u32 vlog_alignment(u32 prev_insn_print_len)
+{
+       if (prev_insn_print_len < BPF_LOG_ALIGNMENT_POS)
+               return BPF_LOG_ALIGNMENT_POS - prev_insn_print_len + 1;

why +1 here?

I wanted the insn_state to be prepended by BPF_LOG_ALIGNMENT_POS number of characters, in this case, that would be prepended by 32 and starting at 33. This ensures that whether prev_insn_print_len is smaller than BPF_LOG_ALIGNMENT_POS or not, the insn_states would be aligned properly, there would be an off-by-one error otherwise.

Quoted 15 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 811

Quoted 12 lines
kernel/bpf/verifier.c +10 -0
+       return round_up(prev_insn_print_len, BPF_LOG_MIN_ALIGNMENT) -
+              prev_insn_print_len;
+}
+
+static void print_prev_insn_state(struct bpf_verifier_env *env,
+                                 const struct bpf_func_state *state)
+{
+       if (env->prev_log_len == env->log.len_used) {
+               if (env->prev_log_len)
+                       bpf_vlog_reset(&env->log, env->prev_log_len - 1);

I don't get this... why do we need this reset? Why just appending alignment spaces below doesn't work?

The insn are printed via print_bpf_insn() which ends the line with a '\n', we need to reset one character in order to remove the new line and print insn_state on the same line. print_bpf_insn() is defined in kernel/bpf/disasm.h and used by bpf_tool as well, so I didn't want to modify it.

oh, can you please leave a small comment explaining that?

Quoted 92 lines
Quoted 92 lines
Quoted 92 lines

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 813

Quoted 4 lines
kernel/bpf/verifier.c +2 -0
+               verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len),
+                       ' ');

nit: keep it on single line

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 815

Quoted 2 lines
kernel/bpf/verifier.c +2 -0
+       } else
+               verbose(env, "%d:", env->insn_idx);

if one branch of if/else has {}, the other one has to have them as well, even if it's a single line statement

Quoted 6 lines
kernel/bpf/verifier.c +3 -0
+       print_verifier_state(env, state);
+}
+
 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  * small to hold src. This is different from krealloc since we don't want to preserve
  * the contents of dst.

[...]

kernel/bpf/verifier.c static int check_cond_jmp_op(struct bpf_verifier_env *env, · line 9470

Quoted 9 lines
kernel/bpf/verifier.c +3 -2
@@ -9441,8 +9465,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
                        insn->dst_reg);
                return -EACCES;
        }
-       if (env->log.level & BPF_LOG_LEVEL)
-               print_verifier_state(env, this_branch->frame[this_branch->curframe]);
+       if (env->log.level & BPF_LOG_LEVEL) {
+               print_prev_insn_state(
+                       env, this_branch->frame[this_branch->curframe]);

nit: keep on a single line. But also is it really a "previous instruction" or still a current instruction? Maybe just "print_insn_state"? Do we have "next_insn" helper anywhere? If not, dropping this "prev_" prefix from helpers and variables would be cleaner, IMO

Quoted 10 lines
kernel/bpf/verifier.c +1 -2
+       }
        return 0;
 }
 
@@ -11310,17 +11336,12 @@ static int do_check(struct bpf_verifier_env *env)
                if (need_resched())
                        cond_resched();
 
-               if ((env->log.level & BPF_LOG_LEVEL2) ||
-                   (env->log.level & BPF_LOG_LEVEL && do_print_state)) {

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11339

Quoted 10 lines
kernel/bpf/verifier.c +1 -9
-                       if (verifier_state_scratched(env) &&
-                           (env->log.level & BPF_LOG_LEVEL2))
-                               verbose(env, "%d:", env->insn_idx);
-                       else
-                               verbose(env, "\nfrom %d to %d%s:",
-                                       env->prev_insn_idx, env->insn_idx,
-                                       env->cur_state->speculative ?
-                                       " (speculative execution)" : "");
-                       print_verifier_state(env, state->frame[state->curframe]);
+               if (env->log.level & BPF_LOG_LEVEL1 && do_print_state) {

() around bit operations

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11344

Quoted 5 lines
kernel/bpf/verifier.c +5 -0
+                       verbose(env, "\nfrom %d to %d%s:\n", env->prev_insn_idx,
+                               env->insn_idx,
+                               env->cur_state->speculative ?
+                                       " (speculative execution)" :
+                                             "");

it's ok to go up to 100 characters, please keep the code more readable

Quoted 21 lines
kernel/bpf/verifier.c +8 -0
                        do_print_state = false;
                }
 
@@ -11331,9 +11352,17 @@ static int do_check(struct bpf_verifier_env *env)
                                .private_data   = env,
                        };
 
+                       if (verifier_state_scratched(env))
+                               print_prev_insn_state(
+                                       env, state->frame[state->curframe]);
+
                        verbose_linfo(env, env->insn_idx, "; ");
+                       env->prev_log_len = env->log.len_used;
                        verbose(env, "%d: ", env->insn_idx);
                        print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
+                       env->prev_insn_print_len =
+                               env->log.len_used - env->prev_log_len;
+                       env->prev_log_len = env->log.len_used;
                }
 
                if (bpf_prog_is_dev_bound(env->prog->aux)) {

[...]

#10 lore raw

PATCH v2 3/3 Only output backtracking information in log level 2

In reply to Christy Lee

Backtracking information is very verbose, don't print it in log level 1 to improve readability.

---
 kernel/bpf/verifier.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/bpf/verifier.c +3 -3
index a8f1426b0367..2cb86972ed35 100644
@@ -2398,7 +2398,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx,
 
 	if (insn->code == 0)
 		return 0;
-	if (env->log.level & BPF_LOG_LEVEL) {
+	if (env->log.level & BPF_LOG_LEVEL2) {
 		verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
 		verbose(env, "%d: ", idx);
 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2656,7 +2656,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
 		DECLARE_BITMAP(mask, 64);
 		u32 history = st->jmp_history_cnt;
 
-		if (env->log.level & BPF_LOG_LEVEL)
+		if (env->log.level & BPF_LOG_LEVEL2)
 			verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
 		for (i = last_idx;;) {
 			if (skip_first) {
@@ -2743,7 +2743,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
 				new_marks = true;
 			reg->precise = true;
 		}
-		if (env->log.level & BPF_LOG_LEVEL) {
+		if (env->log.level & BPF_LOG_LEVEL2) {
 			mark_verifier_state_scratched(env);
 			verbose(env, "parent %s regs=%x stack=%llx marks:",
 				new_marks ? "didn't have" : "already had",
Signature · 1 line
2.30.2
Acked-by on v2 3/3
#11 lore raw

Re: [PATCH v2 bpf-next 3/3] Only output backtracking information in log level 2

Quoted 6 lines · Christy Lee

Backtracking information is very verbose, don't print it in log level 1 to improve readability.

---

LGTM.

Quoted 37 lines
 kernel/bpf/verifier.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/bpf/verifier.c +3 -4
index a8f1426b0367..2cb86972ed35 100644
@@ -2398,7 +2398,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx,
 
        if (insn->code == 0)
                return 0;
-       if (env->log.level & BPF_LOG_LEVEL) {
+       if (env->log.level & BPF_LOG_LEVEL2) {
                verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
                verbose(env, "%d: ", idx);
                print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2656,7 +2656,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
                DECLARE_BITMAP(mask, 64);
                u32 history = st->jmp_history_cnt;
 
-               if (env->log.level & BPF_LOG_LEVEL)
+               if (env->log.level & BPF_LOG_LEVEL2)
                        verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
                for (i = last_idx;;) {
                        if (skip_first) {
@@ -2743,7 +2743,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
                                new_marks = true;
                        reg->precise = true;
                }
-               if (env->log.level & BPF_LOG_LEVEL) {
+               if (env->log.level & BPF_LOG_LEVEL2) {
                        mark_verifier_state_scratched(env);
                        verbose(env, "parent %s regs=%x stack=%llx marks:",
                                new_marks ? "didn't have" : "already had",
--

2.30.2