add bpf_stale scheduler
Subsystems (4)
v2 cover add bpf_stale scheduler
v2:
- store subflow ids instead of storing subflow pointers in sk_storage.
v1:
- This patchset adds the new bpf_stale scheduler. Use sk_storage to save the stale map instead of using subflow->stale flag to manage it.
Geliang Tang (2):
selftests/bpf: Add bpf_stale scheduler
selftests/bpf: Add bpf_stale test
tools/testing/selftests/bpf/bpf_tcp_helpers.h | 1 + .../testing/selftests/bpf/prog_tests/mptcp.c | 38 +++++ .../selftests/bpf/progs/mptcp_bpf_stale.c | 161 ++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c
Signature · 1 line
2.35.3
PATCH v2 1/2 selftests/bpf: Add bpf_stale scheduler
This patch implements the setting a subflow as stale/unstale in BPF MPTCP scheduler, named bpf_stale. The staled subflow id will be added into a map in sk_storage.
Two helper mptcp_subflow_set_stale() and mptcp_subflow_clear_stale() are added.
In this test, subflow 1 is set as stale in bpf_stale_data_init(). Each subflow is checked whether it's a stale one in bpf_stale_get_subflow() to select a unstale subflow to send data.
--- tools/testing/selftests/bpf/bpf_tcp_helpers.h | 1 + .../selftests/bpf/progs/mptcp_bpf_stale.c | 161 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c
tools/testing/selftests/bpf/bpf_tcp_helpers.h +1 -0
@@ -239,6 +239,7 @@ struct mptcp_subflow_context {unsigned long avg_pacing_rate; __u32 backup : 1;__u8 stale_count; +__u32 subflow_id;struct sock *tcp_sock;/* tcp sk backpointer */ } __attribute__((preserve_access_index));
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +161 -0
161 lines not on this page · raw message
Signature · 1 line
2.35.3
Mat Martineau
maintainerRe: [PATCH mptcp-next v2 1/2] selftests/bpf: Add bpf_stale scheduler
Quoted 111 lines · Geliang Tang
This patch implements the setting a subflow as stale/unstale in BPF MPTCP scheduler, named
bpf_stale. The staled subflow id will be added into a map insk_storage.Two helper
mptcp_subflow_set_stale()andmptcp_subflow_clear_stale()are added.In this test, subflow 1 is set as stale in
bpf_stale_data_init(). Each subflow is checked whether it's a stale one inbpf_stale_get_subflow()to select a unstale subflow to send data.Signed-off-by Geliang Tang <[email protected]>---
tools/testing/selftests/bpf/bpf_tcp_helpers.h| 1 +.../selftests/bpf/progs/mptcp_bpf_stale.c| 161 ++++++++++++++++++2 files changed, 162 insertions(+)create mode 100644
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.ctools/testing/selftests/bpf/bpf_tcp_helpers.h +1
-0@@ -239,6 +239,7 @@ struct mptcp_subflow_context {unsigned long avg_pacing_rate; __u32 backup : 1;__u8 stale_count; +__u32 subflow_id;struct sock *tcp_sock;/* tcp sk backpointer */ } __attribute__((preserve_access_index));tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +75
-0@@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023, SUSE. */ + +#include <linux/bpf.h> +#include "bpf_tcp_helpers.h" + +char _license[] SEC("license") = "GPL"; + +struct mptcp_stale_storage { +__u8 nr; +__u32 ids[MPTCP_SUBFLOWS_MAX ]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct mptcp_stale_storage); +} mptcp_stale_map SEC(".maps"); + +static void mptcp_subflow_set_stale (struct mptcp_stale_storage *storage, +__u32 subflow_id) +{ +if (!subflow_id) +return ; + +for (int i =0 ; i < storage->nr && i <MPTCP_SUBFLOWS_MAX ; i++) { +if (storage->ids[i] == subflow_id) +return ; + } + +if (storage->nr <MPTCP_SUBFLOWS_MAX -1 ) + storage->ids[storage->nr++] = subflow_id; +} + +static void mptcp_subflow_clear_stale (struct mptcp_stale_storage *storage, +__u32 subflow_id) +{ +if (!subflow_id) +return ; + +for (int i =0 ; i < storage->nr && i <MPTCP_SUBFLOWS_MAX ; i++) { +if (storage->ids[i] == subflow_id) { +for (int j = i; j <MPTCP_SUBFLOWS_MAX -1 ; j++) { +if (!storage->ids[j +1 ]) +break ; + storage->ids[j] = storage->ids[j +1 ]; + storage->ids[j +1 ] =0 ; + } + storage->nr--; +return ; + } + } +} + +static bool mptcp_subflow_is_stale (struct mptcp_stale_storage *storage, +__u32 subflow_id) +{ +for (int i =0 ; i < storage->nr && i <MPTCP_SUBFLOWS_MAX ; i++) { +if (storage->ids[i] == subflow_id) +return true; + } + +return false; +} + +static bool mptcp_subflow_is_active (struct mptcp_sched_data *data, +__u32 subflow_id) +{ +for (int i =0 ; i < data->subflows && i <MPTCP_SUBFLOWS_MAX ; i++) { +struct mptcp_subflow_context *subflow; + + subflow =mptcp_subflow_ctx_by_pos (data, i); +if (!subflow) +break ;
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 85
Quoted 10 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +10
-0+if (subflow->subflow_id == subflow_id) +return true; + } + +return false; +} + +SEC("struct_ops/mptcp_sched_stale_init") +void BPF_PROG (mptcp_sched_stale_init ,struct mptcp_sock *msk) +{
It looks like the sk_storage should be initialized here so it can be set up once, not every time the scheduler is called.
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 90
Quoted 5 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +5
-0+} + +SEC("struct_ops/mptcp_sched_stale_release") +void BPF_PROG (mptcp_sched_stale_release ,struct mptcp_sock *msk) +{
Should this release the sk_storage?
Quoted 9 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +9
-0+} + +void BPF_STRUCT_OPS (bpf_stale_data_init ,struct mptcp_sock *msk, +struct mptcp_sched_data *data) +{ +struct mptcp_subflow_context *subflow; +struct mptcp_stale_storage *storage; + +mptcp_sched_data_set_contexts (msk, data);
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 109
Quoted 10 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +10
-0+ + storage =bpf_sk_storage_get (&mptcp_stale_map, msk,0 , +BPF_LOCAL_STORAGE_GET_F_CREATE ); +if (!storage) +return ; + +for (int i =0 ; i < storage->nr && i <MPTCP_SUBFLOWS_MAX ; i++) { +if (!mptcp_subflow_is_active (data, storage->ids[i])) +mptcp_subflow_clear_stale (storage, storage->ids[i]); + }
Why is this loop here? Is it to handle invalid subflow ids for subflows that have been closed?
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 114
Quoted 5 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +5
-0+ + subflow =mptcp_subflow_ctx_by_pos (data,0 ); +if (subflow) { +mptcp_subflow_set_stale (storage, subflow->subflow_id); +mptcp_subflow_clear_stale (storage, subflow->subflow_id);
Why set and then clear?
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 121
Quoted 7 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +7
-0+ } + + subflow =mptcp_subflow_ctx_by_pos (data,1 ); +if (subflow) { +mptcp_subflow_set_stale (storage, subflow->subflow_id); +mptcp_subflow_clear_stale (storage, subflow->subflow_id); +mptcp_subflow_set_stale (storage, subflow->subflow_id);
Same question for set/clear/set
Quoted 20 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +20
-0+ } +} + +int BPF_STRUCT_OPS (bpf_stale_get_subflow ,struct mptcp_sock *msk, +const struct mptcp_sched_data *data) +{ +struct mptcp_stale_storage *storage; +int nr =-1 ; + + storage =bpf_sk_storage_get (&mptcp_stale_map, msk,0 , +BPF_LOCAL_STORAGE_GET_F_CREATE ); +if (!storage) +return -1 ; + +for (int i =0 ; i < data->subflows && i <MPTCP_SUBFLOWS_MAX ; i++) { +struct mptcp_subflow_context *subflow; + + subflow =mptcp_subflow_ctx_by_pos (data, i); +if (!subflow) +break ;
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c line 151
Quoted 10 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +10
-0+ +if (mptcp_subflow_is_stale (storage, subflow->subflow_id)) +continue ; + + nr = i; + } + +if (nr !=-1 ) +mptcp_subflow_set_scheduled (mptcp_subflow_ctx_by_pos (data, nr), true); +return 0 ;
I missed this detail on a previous review: if no subflow is scheduled, it should return '1' to indicate an error.
Quoted 15 lines
tools/testing/selftests/bpf/progs/mptcp_bpf_stale.c new file +10
-0+} + +SEC(".struct_ops") +struct mptcp_sched_ops stale = { + .init = (void *)mptcp_sched_stale_init, + .release = (void *)mptcp_sched_stale_release, + .data_init = (void *)bpf_stale_data_init, + .get_subflow = (void *)bpf_stale_get_subflow, + .name ="bpf_stale" , +};Signature · 1 line
2.35.3
PATCH v2 2/2 selftests/bpf: Add bpf_stale test
In reply to Geliang TangThis patch adds the bpf_stale scheduler test: test_stale(). Use sysctl to set net.mptcp.scheduler to use this sched. Add two veth net devices to simulate the multiple addresses case. Use 'ip mptcp endpoint' command to add the new endpoint ADDR_2 to PM netlink. Send data and check bytes_sent of 'ss' output after it to make sure the data has been only sent on ADDR_1 since ADDR_2 is set as stale.
--- .../testing/selftests/bpf/prog_tests/mptcp.c | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+)
tools/testing/selftests/bpf/prog_tests/mptcp.c +38 -0
@@ -12,6 +12,7 @@#include "mptcp_bpf_rr.skel.h" #include "mptcp_bpf_red.skel.h" #include "mptcp_bpf_burst.skel.h" +#include "mptcp_bpf_stale.skel.h" char NS_TEST [32 ];
@@ -524,6 +525,41 @@ static void test_burst(void)mptcp_bpf_burst__destroy (burst_skel); } +static void test_stale (void ) +{ +struct mptcp_bpf_stale *stale_skel; +int server_fd, client_fd; +struct nstoken *nstoken; +struct bpf_link *link; + + stale_skel =mptcp_bpf_stale__open_and_load (); +if (!ASSERT_OK_PTR (stale_skel,"bpf_stale__open_and_load" )) +return ; + + link =bpf_map__attach_struct_ops (stale_skel->maps.stale); +if (!ASSERT_OK_PTR (link,"bpf_map__attach_struct_ops" )) { +mptcp_bpf_stale__destroy (stale_skel); +return ; + } + + nstoken =sched_init ("subflow" ,"bpf_stale" ); +if (!ASSERT_OK_PTR (nstoken,"sched_init:bpf_stale" )) + goto fail; + server_fd =start_mptcp_server (AF_INET ,ADDR_1 ,PORT_1 ,0 ); + client_fd =connect_to_fd (server_fd,0 ); + +send_data (server_fd, client_fd,"bpf_stale" ); +ASSERT_OK (has_bytes_sent (ADDR_1 ),"has_bytes_sent addr_1" ); +ASSERT_GT (has_bytes_sent (ADDR_2 ),0 ,"has_bytes_sent addr_2" ); + +close (client_fd); +close (server_fd); +fail: +cleanup_netns (nstoken); +bpf_link__destroy (link); +mptcp_bpf_stale__destroy (stale_skel); +} +void test_mptcp (void ) {if (test__start_subtest ("base" ))
@@ -540,4 +576,6 @@ void test_mptcp(void)test_red ();if (test__start_subtest ("burst" ))test_burst (); +if (test__start_subtest ("stale" )) +test_stale (); }
Signature · 1 line
2.35.3
Re: selftests/bpf: Add bpf_stale test: Tests Results
Hi Geliang,
Thank you for your modifications, that's great!
Our CI did some validations and here is its report:
- KVM Validation: normal (except selftest_mptcp_join):
- Success! ✅:
- Task: https://cirrus-ci.com/task/5811024205447168
- Summary: https://api.cirrus-ci.com/v1/artifact/task/5811024205447168/summary/summary.txt
- KVM Validation: normal (only selftest_mptcp_join):
- Success! ✅:
- Task: https://cirrus-ci.com/task/5248074252025856
- Summary: https://api.cirrus-ci.com/v1/artifact/task/5248074252025856/summary/summary.txt
- KVM Validation: debug (only selftest_mptcp_join):
- Success! ✅:
- Task: https://cirrus-ci.com/task/4966599275315200
- Summary: https://api.cirrus-ci.com/v1/artifact/task/4966599275315200/summary/summary.txt
- KVM Validation: debug (except selftest_mptcp_join):
- Unstable: 1 failed test(s): packetdrill_mp_join 🔴:
- Task: https://cirrus-ci.com/task/6373974158868480
- Summary: https://api.cirrus-ci.com/v1/artifact/task/6373974158868480/summary/summary.txt
Initiator: Patchew Applier
Commits: https://github.com/multipath-tcp/mptcp_net-next/commits/2174a36d15db
If there are some issues, you can reproduce them using the same environment as the one used by the CI thanks to a docker image, e.g.:
$ cd [kernel source code]
$ docker run -v "${PWD}:${PWD}:rw" -w "${PWD}" --privileged --rm -it \
--pull always mptcp/mptcp-upstream-virtme-docker:latest \
auto-debugFor more details:
https://github.com/multipath-tcp/mptcp-upstream-virtme-docker
Please note that despite all the efforts that have been already done to have a stable tests suite when executed on a public CI like here, it is possible some reported issues are not due to your modifications. Still, do not hesitate to help us improve that ;-)
Cheers,
MPTCP GH Action bot
Bot operated by Matthieu Baerts (Tessares)