Skip to contentSkip to conversation

Improve verifier log readability

PATCH In review Christy Lee 3 participants 7 messages Dec 16, 2021 – Dec 17, 2021

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

v3 cover Improve verifier log readability

Simplify verifier logs and improve readability.

Changelog:
----------
v2 -> v3:
v2: https://lore.kernel.org/all/[email protected]/

Patch 1:
* Fixed typo
* Added print_all bool arg to print_verifier_state()

Patch 2:
* Changed alignment from 32 to 40, fixed off-by-one error
* Renamed print_prev_insn_state() to print_insn_state()
* Fixed formatting to make the code more readable

v1 -> v2:
v1: https://lore.kernel.org/bpf/[email protected]/

Patch 2/3:
* Verifier will skip insn_idx when the insn is longer than 8 bytes, example:
0000000000000000 <good_prog>:

       0:	18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00	r2 = 0 ll
       2:	63 12 00 00 00 00 00 00	*(u32 *)(r2 + 0) = r1
       3:	61 20 04 00 00 00 00 00	r0 = *(u32 *)(r2 + 4)
       4:	95 00 00 00 00 00 00 00	exit

It's incorrect to check that prev_insn_idx = insn_idx - 1, skip this check and print the verifier state on the correct line.
Before:
0: R1=ctx(id=0,off=0,imm=0) R10=fp0
; a[0] = (int)(long)ctx;
0: (18) r2 = 0xffffc900006de000
2: R2_w=map_value(id=0,off=0,ks=4,vs=16,imm=0)
2: (63) *(u32 *)(r2 +0) = r1
After:
0: R1=ctx(id=0,off=0,imm=0) R10=fp0
; a[0] = (int)(long)ctx;
0: (18) r2 = 0xffffc900006de000 ; R2_w=map_value(id=0,off=0,ks=4,vs=16,imm=0) 2: (63) *(u32 *)(r2 +0) = r1
* Track previous line logging length in env, allow aligning intsruction from anywhere in the verifier
* Fixed bug where the verifier printed verifier state after checking source memory access but before check destination memory access, this ensures the aligned verifier state contains all scratched registers

Patch 3/3:
* Added one more case where we should only log in log_level=2

Christy Lee (3):
Only print scratched registers and stack slots to verifier logs Right align verifier states in verifier logs
Only output backtracking information in log level 2

 include/linux/bpf_verifier.h                  |  10 +
 kernel/bpf/verifier.c                         | 130 ++++++++---
 .../testing/selftests/bpf/prog_tests/align.c  | 214 +++++++++---------
 3 files changed, 225 insertions(+), 129 deletions(-)

--
2.30.2

#1 lore raw

PATCH v3 1/3 Only print scratched registers and stack slots to verifier logs

When printing verifier state for any log level, print full verifier state only on function calls or on errors. Otherwise, only print the registers and stack slots that were accessed.

Log size differences:

verif_scale_loop6 before: 234566564
verif_scale_loop6 after: 72143943
69% size reduction

kfree_skb before: 166406
kfree_skb after: 55386
69% size reduction

Before:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=invP0 R10=fp0 fp-8_w=00000000 fp-16_w=00\
000000 fp-24_w=00000000 fp-32_w=00000000 fp-40_w=00000000 fp-48_w=00000000 fp-56_w=00000000 fp-64_w=00000000 fp-72_w=00000000 fp-80_w=00000\
000 fp-88_w=00000000 fp-96_w=00000000 fp-104_w=00000000 fp-112_w=00000000 fp-120_w=00000000 fp-128_w=00000000 fp-136_w=00000000 fp-144_w=00\
000000 fp-152_w=00000000 fp-160_w=00000000 fp-168_w=00000000 fp-176_w=00000000 fp-184_w=00000000 fp-192_w=00000000 fp-200_w=00000000 fp-208\
_w=00000000 fp-216_w=00000000 fp-224_w=00000000 fp-232_w=00000000 fp-240_w=00000000 fp-248_w=00000000 fp-256_w=00000000 fp-264_w=00000000 f\
p-272_w=00000000 fp-280_w=00000000 fp-288_w=00000000 fp-296_w=00000000 fp-304_w=00000000 fp-312_w=00000000 fp-320_w=00000000 fp-328_w=00000\
000 fp-336_w=00000000 fp-344_w=00000000 fp-352_w=00000000 fp-360_w=00000000 fp-368_w=00000000 fp-376_w=00000000 fp-384_w=00000000 fp-392_w=\
00000000 fp-400_w=00000000 fp-408_w=00000000 fp-416_w=00000000 fp-424_w=00000000 fp-432_w=00000000 fp-440_w=00000000 fp-448_w=00000000 ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1) R10=fp0
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=1) R10=fp0
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0

After:

156: (61) r0 = *(u32 *)(r1 +0)
157: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) ; return skb->len;
157: (95) exit
Func#4 is safe for any args that match its prototype
Validating get_constant() func#5...
158: R1=invP(id=0) R10=fp0
; int get_constant(long val)
158: (bf) r0 = r1
159: R0_w=invP(id=1) R1=invP(id=1)
; return val - 122;
159: (04) w0 += -122
160: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return val - 122;
160: (95) exit
Func#5 is safe for any args that match its prototype
Validating get_skb_ifindex() func#6...
161: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R3=invP(id=0) R10=fp0
; int get_skb_ifindex(int val, struct __sk_buff *skb, int var)
161: (bc) w0 = w3
162: R0_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R3=invP(id=0)

---
 include/linux/bpf_verifier.h                  |  7 ++
 kernel/bpf/verifier.c                         | 81 ++++++++++++++++---
 .../testing/selftests/bpf/prog_tests/align.c  | 30 +++----
 3 files changed, 90 insertions(+), 28 deletions(-)
include/linux/bpf_verifier.h +7 -0
index 182b16a91084..c66f238c538d 100644
@@ -474,6 +474,13 @@ struct bpf_verifier_env {
 	/* longest register parentage chain walked for liveness marking */
 	u32 longest_mark_read_walk;
 	bpfptr_t fd_array;
+
+	/* bit mask to keep track of whether a register has been accessed
+	 * since the last time the function state was printed
+	 */
+	u32 scratched_regs;
+	/* Same as scratched_regs but for stack slots */
+	u64 scratched_stack_slots;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
kernel/bpf/verifier.c +68 -13

202 lines not on this page · raw message

tools/testing/selftests/bpf/prog_tests/align.c +15 -15
index 837f67c6bfda..aeb2080a67f7 100644
@@ -39,8 +39,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv2"},
 			{2, "R3_w=inv4"},
 			{3, "R3_w=inv8"},
@@ -67,8 +67,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv1"},
 			{2, "R3_w=inv2"},
 			{3, "R3_w=inv4"},
@@ -96,8 +96,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv4"},
 			{2, "R3_w=inv8"},
 			{3, "R3_w=inv10"},
@@ -118,8 +118,8 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{1, "R1=ctx(id=0,off=0,imm=0)"},
-			{1, "R10=fp0"},
+			{0, "R1=ctx(id=0,off=0,imm=0)"},
+			{0, "R10=fp0"},
 			{1, "R3_w=inv7"},
 			{2, "R3_w=inv7"},
 			{3, "R3_w=inv14"},
@@ -161,13 +161,13 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{7, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
+			{6, "R0_w=pkt(id=0,off=8,r=8,imm=0)"},
 			{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
 			{9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			{10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
 			{11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
-			{18, "R3=pkt_end(id=0,off=0,imm=0)"},
+			{13, "R3_w=pkt_end(id=0,off=0,imm=0)"},
 			{18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
 			{20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
@@ -234,10 +234,10 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
+			{3, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
 			{5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
 			{6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
-			{10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
+			{9, "R2=pkt(id=0,off=0,r=18,imm=0)"},
 			{10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
 			{10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
 			{14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
@@ -296,7 +296,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+			{6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
 			{8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			/* Offset is added to packet pointer R5, resulting in
 			 * known fixed offset, and variable offset from R6.
@@ -386,7 +386,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{8, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
+			{6, "R2_w=pkt(id=0,off=0,r=8,imm=0)"},
 			{8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
 			/* Adding 14 makes R6 be (4n+2) */
 			{9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
@@ -458,7 +458,7 @@ static struct bpf_align_test tests[] = {
 			/* Checked s>=0 */
 			{9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			/* packet pointer + nonnegative (4n+2) */
-			{11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
+			{12, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			{13, "R4_w=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc)"},
 			/* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine.
 			 * We checked the bounds, but it might have been able
Signature · 1 line
2.30.2
#2 lore raw

Re: [PATCH v3 bpf-next 1/3] Only print scratched registers and stack slots to verifier logs

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11309

Quoted 6 lines · Christy Lee
kernel/bpf/verifier.c +1 -1
@@ -11256,16 +11306,18 @@ static int do_check(struct bpf_verifier_env *env)
                if (need_resched())
                        cond_resched();
 
-               if (env->log.level & BPF_LOG_LEVEL2 ||
+               if ((env->log.level & BPF_LOG_LEVEL2) ||

Why add redundant () ?

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11312

Quoted 4 lines
kernel/bpf/verifier.c +2 -1
                    (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
-                       if (env->log.level & BPF_LOG_LEVEL2)
+                       if (verifier_state_scratched(env) &&
+                           (env->log.level & BPF_LOG_LEVEL2))

redundant ()

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11318

Quoted 6 lines
kernel/bpf/verifier.c +0 -0
                                verbose(env, "%d:", env->insn_idx);
                        else
                                verbose(env, "\nfrom %d to %d%s:",
                                        env->prev_insn_idx, env->insn_idx,
                                        env->cur_state->speculative ?
                                        " (speculative execution)" : "");

Also the addition of "verifier_state_scratched(env) &&" changes the logic.
Instead of printing "%d:" it prints "from %d to %d" from time to time which looks really weird.
The "from %d to %d" used to be printed only when
log_level == 1 && do_print_state==true.
It's doing this to indicate that the state was popped from the stack. It's a branch processing.
When "from %d to %d" appears in the middle of the basic block it's confusing.
When vlog_reset logic went in the "from %d to %d" became irrelevant, since they would never be seen anymore,
but for folks who've seen the logs earlier it's odd.

I think it meant to be:

                if (env->log.level & BPF_LOG_LEVEL2 ||
                    (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
                        if (env->log.level & BPF_LOG_LEVEL2) {
                                if (verifier_state_scratched(env))
                                        verbose(env, "%d:", env->insn_idx);
                        } else {
                                verbose(env, "\nfrom %d to %d%s:",
                                        env->prev_insn_idx, env->insn_idx,
                                        env->cur_state->speculative ?
                                        " (speculative execution)" : "");
                        }
                        print_verifier_state(env, state->frame[state->curframe],
                                             false);
                        do_print_state = false;
                }

It's all minor. I fixed it all up while applying.

#3 lore raw

PATCH v3 2/3 Right align verifier states in verifier logs

In reply to Christy Lee

Make the verifier logs more readable, print the verifier states on the corresponding instruction line. If the previous line was not a bpf instruction, then print the verifier states on its own line.

Before:

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2
87: R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1
88: R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6
89: R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0
91: R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123
92: R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

After:

86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2 ; R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1 ; R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
88: (bf) r1 = r6 ; R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0 ; R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
91: (b7) r1 = 123 ; R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

---
 include/linux/bpf_verifier.h                  |   3 +
 kernel/bpf/verifier.c                         |  61 ++++--
 .../testing/selftests/bpf/prog_tests/align.c  | 196 ++++++++++--------
 3 files changed, 147 insertions(+), 113 deletions(-)
include/linux/bpf_verifier.h +3 -0
index c66f238c538d..ee931398f311 100644
@@ -388,6 +388,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
 #define BPF_LOG_LEVEL	(BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
 #define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS)
 #define BPF_LOG_KERNEL	(BPF_LOG_MASK + 1) /* kernel internal flag */
+#define BPF_LOG_MIN_ALIGNMENT 8U
+#define BPF_LOG_ALIGNMENT 40U
 
 static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 {
@@ -481,6 +483,7 @@ struct bpf_verifier_env {
 	u32 scratched_regs;
 	/* Same as scratched_regs but for stack slots */
 	u64 scratched_stack_slots;
+	u32 prev_log_len, prev_insn_print_len;
 };
 
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
kernel/bpf/verifier.c +40 -21

114 lines not on this page · raw message

tools/testing/selftests/bpf/prog_tests/align.c +104 -92

368 lines not on this page · raw message

Signature · 1 line
2.30.2
Acked-by on v3 2/3
#4 lore raw

Re: [PATCH v3 bpf-next 2/3] Right align verifier states in verifier logs

Quoted 48 lines · Christy Lee

Make the verifier logs more readable, print the verifier states on the corresponding instruction line. If the previous line was not a bpf instruction, then print the verifier states on its own line.

Before:

Validating test_pkt_access_subprog3() func#3...
86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2
87: R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1
88: R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 88: (bf) r1 = r6
89: R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0
91: R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) ; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123)); 91: (b7) r1 = 123
92: R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

After:

86: R1=invP(id=0) R2=ctx(id=0,off=0,imm=0) R10=fp0
; int test_pkt_access_subprog3(int val, struct __sk_buff *skb)
86: (bf) r6 = r2 ; R2=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
87: (bc) w7 = w1 ; R1=invP(id=0) R7_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
88: (bf) r1 = r6 ; R1_w=ctx(id=0,off=0,imm=0) R6_w=ctx(id=0,off=0,imm=0)
89: (85) call pc+9
Func#4 is global and valid. Skipping.
90: R0_w=invP(id=0)
90: (bc) w8 = w0 ; R0_w=invP(id=0) R8_w=invP(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff))
; return get_skb_len(skb) * get_skb_ifindex(val, skb, get_constant(123));
91: (b7) r1 = 123 ; R1_w=invP123
92: (85) call pc+65
Func#5 is global and valid. Skipping.
93: R0=invP(id=0)

Quoted 7 lines
---
 include/linux/bpf_verifier.h                  |   3 +
 kernel/bpf/verifier.c                         |  61 ++++--
 .../testing/selftests/bpf/prog_tests/align.c  | 196 ++++++++++--------
 3 files changed, 147 insertions(+), 113 deletions(-)

Checked pyperf50 output, looks great. Thanks for improving verifier logs!

Quoted 11 lines
include/linux/bpf_verifier.h +2 -0
index c66f238c538d..ee931398f311 100644
@@ -388,6 +388,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
 #define BPF_LOG_LEVEL  (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
 #define BPF_LOG_MASK   (BPF_LOG_LEVEL | BPF_LOG_STATS)
 #define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
+#define BPF_LOG_MIN_ALIGNMENT 8U
+#define BPF_LOG_ALIGNMENT 40U
 

[...]

#5 lore raw

Re: [PATCH v3 bpf-next 2/3] Right align verifier states in verifier logs

In reply to Christy Lee

kernel/bpf/verifier.c static void print_verifier_state(struct bpf_verifier_env *env, · line 808

Quoted 4 lines · Christy Lee
kernel/bpf/verifier.c +4 -0
+static void print_insn_state(struct bpf_verifier_env *env,
+                            const struct bpf_func_state *state)
+{
+       if (env->prev_log_len && (env->prev_log_len == env->log.len_used)) {

redundant ()

Quoted 1 line

+ /* remove new line character*/

missing ' ' before */

Quoted 48 lines
kernel/bpf/verifier.c +14 -7
+               bpf_vlog_reset(&env->log, env->prev_log_len - 1);
+               verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' ');
+       } else {
+               verbose(env, "%d:", env->insn_idx);
+       }
+       print_verifier_state(env, state, false);
+}
+
 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  * small to hold src. This is different from krealloc since we don't want to preserve
  * the contents of dst.
@@ -2724,10 +2743,10 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
                        reg->precise = true;
                }
                if (env->log.level & BPF_LOG_LEVEL) {
-                       print_verifier_state(env, func, false);
-                       verbose(env, "parent %s regs=%x stack=%llx marks\n",
+                       verbose(env, "parent %s regs=%x stack=%llx marks:",
                                new_marks ? "didn't have" : "already had",
                                reg_mask, stack_mask);
+                       print_verifier_state(env, func, true);
                }
 
                if (!reg_mask && !stack_mask)
@@ -3422,11 +3441,8 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
        /* We may have adjusted the register pointing to memory region, so we
         * need to try adding each of min_value and max_value to off
         * to make sure our theoretical access will be safe.
-        */
-       if (env->log.level & BPF_LOG_LEVEL)
-               print_verifier_state(env, state, false);
-
-       /* The minimum value is only important with signed
+        *
+        * The minimum value is only important with signed
         * comparisons where we can't assume the floor of a
         * value is 0.  If we are using signed variables for our
         * index'es we need to make sure that whatever we use
@@ -4565,6 +4581,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 
 static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
 {
+       struct bpf_verifier_state *vstate = env->cur_state;
+       struct bpf_func_state *state = vstate->frame[vstate->curframe];
        int load_reg;
        int err;
 

kernel/bpf/verifier.c static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · line 4674

Quoted 6 lines
kernel/bpf/verifier.c +3 -0
@@ -4651,6 +4669,9 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i
        if (err)
                return err;
 
+       if (env->log.level & BPF_LOG_LEVEL)
+               print_insn_state(env, state);
+

I couldn't figure out why check_atomic() is special when the main loop in do_check() is doing it anyway. So I've removed this hunk and one above.

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11329

Quoted 13 lines
kernel/bpf/verifier.c +1 -12
-               if ((env->log.level & BPF_LOG_LEVEL2) ||
-                   (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
-                       if (verifier_state_scratched(env) &&
-                           (env->log.level & BPF_LOG_LEVEL2))
-                               verbose(env, "%d:", env->insn_idx);
-                       else
-                               verbose(env, "\nfrom %d to %d%s:",
-                                       env->prev_insn_idx, env->insn_idx,
-                                       env->cur_state->speculative ?
-                                       " (speculative execution)" : "");
-                       print_verifier_state(env, state->frame[state->curframe],
-                                            false);
+               if ((env->log.level & BPF_LOG_LEVEL1) && do_print_state) {

redundant ()

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11332

Quoted 3 lines
kernel/bpf/verifier.c +3 -0
+                       verbose(env, "\nfrom %d to %d%s:\n", env->prev_insn_idx,
+                               env->insn_idx, env->cur_state->speculative ?
+                               " (speculative execution)" : "");

Due to vlog_reset at log_level=1 "from %d to %d"
is not seen anymore.
So it should be LEVEL2 instead of LEVEL1.
Then this 'from %d to %d' becomes meaningful in full verifier log.

kernel/bpf/verifier.c static int do_check(struct bpf_verifier_env *env) · line 11351

Quoted 13 lines
kernel/bpf/verifier.c +6 -0
@@ -11328,9 +11340,16 @@ static int do_check(struct bpf_verifier_env *env)
                                .private_data   = env,
                        };
 
+                       if (verifier_state_scratched(env))
+                               print_insn_state(env, state->frame[state->curframe]);
+
                        verbose_linfo(env, env->insn_idx, "; ");
+                       env->prev_log_len = env->log.len_used;
                        verbose(env, "%d: ", env->insn_idx);
                        print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
+                       env->prev_insn_print_len =
+                               env->log.len_used - env->prev_log_len;

no need to wrap the line.

I fixed it all up while applying.

#6 lore raw

PATCH v3 3/3 Only output backtracking information in log level 2

In reply to Christy Lee

Backtracking information is very verbose, don't print it in log level 1 to improve readability.

---
 kernel/bpf/verifier.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
kernel/bpf/verifier.c +3 -3
index 89d0026e3b17..a586d622d6f5 100644
@@ -2397,7 +2397,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx,
 
 	if (insn->code == 0)
 		return 0;
-	if (env->log.level & BPF_LOG_LEVEL) {
+	if (env->log.level & BPF_LOG_LEVEL2) {
 		verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
 		verbose(env, "%d: ", idx);
 		print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
@@ -2655,7 +2655,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
 		DECLARE_BITMAP(mask, 64);
 		u32 history = st->jmp_history_cnt;
 
-		if (env->log.level & BPF_LOG_LEVEL)
+		if (env->log.level & BPF_LOG_LEVEL2)
 			verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
 		for (i = last_idx;;) {
 			if (skip_first) {
@@ -2742,7 +2742,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
 				new_marks = true;
 			reg->precise = true;
 		}
-		if (env->log.level & BPF_LOG_LEVEL) {
+		if (env->log.level & BPF_LOG_LEVEL2) {
 			verbose(env, "parent %s regs=%x stack=%llx marks:",
 				new_marks ? "didn't have" : "already had",
 				reg_mask, stack_mask);
Signature · 1 line
2.30.2