Skip to contentSkip to conversation

BPF round-robin scheduler

PATCH Superseded Geliang Tang 3 participants 6 messages May 9, 2022 – May 11, 2022

Subsystems (5)

v13 of 15 Superseded Latest is v15 →

lore
Share & tools
lore
#0 lore raw

v13 cover BPF round-robin scheduler

v13:
- add !msk->last_snd check in patch 2
- use ASSERT_OK_PTR instead of CHECK in patch 3
- base-commit: export/20220509T115202

v12:
- init ssk from data->contexts[0], instead of msk->first.
- cycle through all the subflows, instead of the first two.

v11:
- rename array to contexts.
- drop number of subflows in mptcp_sched_data.
- set unused array elements to NULL.
- add MPTCP_SUBFLOWS_MAX check in mptcp_sched_data_init.

v10:
- init subflows array in mptcp_sched_data_init.
- for (int i = 0; i < data->subflows; i++) is not allowed in BPF, using this instead:

	for (int i = 0; i < MPTCP_SUBFLOWS_MAX && i < data->subflows; i++)

- deponds on: "BPF packet scheduler" series v18.

v9:
- add subflows array in mptcp_sched_data
- deponds on: "BPF packet scheduler" series v17 + Squash to "mptcp: add struct mptcp_sched_ops v17".

v8:
- use struct mptcp_sched_data.
- deponds on: "BPF packet scheduler" series v14.

v7:
- rename retrans to reinject.
- drop last_snd setting.
- deponds on: "BPF packet scheduler" series v13.

v6:
- set call_me_again flag.
- deponds on: "BPF packet scheduler" series v12.

v5:
- update patch 2, use temporary storage instead.
- update patch 3, use new helpers.
- deponds on: "BPF packet scheduler" series v11.

v4:
- add retrans argment for get_subflow()

v3:
- add last_snd write access.
- keep msk->last_snd setting in get_subflow().
- deponds on: "BPF packet scheduler" series v10.

v2:
- merge the squash-to patch.
- implement bpf_mptcp_get_subflows helper, instead of bpf_mptcp_get_next_subflow.
- deponds on: "BPF packet scheduler v9".

This patchset implements round-robin scheduler using BPF. Address to some commends for the RFC version:

https://patchwork.kernel.org/project/mptcp/cover/[email protected]/

Geliang Tang (3):
mptcp: add subflows array in sched data
selftests/bpf: add bpf_rr scheduler
selftests/bpf: add bpf_rr test

 include/net/mptcp.h                           |  2 +
 net/mptcp/sched.c                             | 14 ++++++
 .../testing/selftests/bpf/bpf_mptcp_helpers.h |  8 ++++
 .../testing/selftests/bpf/prog_tests/mptcp.c  | 38 +++++++++++++++
 .../selftests/bpf/progs/mptcp_bpf_rr.c        | 47 +++++++++++++++++++
 5 files changed, 109 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/mptcp_bpf_rr.c
Signature · 1 line
2.34.1
#2 lore raw

PATCH v13 2/3 selftests/bpf: add bpf_rr scheduler

This patch implements the round-robin BPF MPTCP scheduler, named bpf_rr, which always picks the next available subflow to send data. If no such next subflow available, picks the first one.

---
 .../testing/selftests/bpf/bpf_mptcp_helpers.h |  6 +++
 .../selftests/bpf/progs/mptcp_bpf_rr.c        | 47 +++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/mptcp_bpf_rr.c
tools/testing/selftests/bpf/bpf_mptcp_helpers.h +6 -0
index 7ecd0b14666a..2d5109f459b4 100644
@@ -29,10 +29,16 @@ struct mptcp_sched_ops {
 struct mptcp_sock {
 	struct inet_connection_sock	sk;
 
+	struct sock	*last_snd;
 	__u32		token;
 	struct sock	*first;
 	struct mptcp_sched_ops *sched;
 	char		ca_name[TCP_CA_NAME_MAX];
 } __attribute__((preserve_access_index));
 
+struct mptcp_subflow_context {
+	__u32	token;
+	struct	sock *tcp_sock;	    /* tcp sk backpointer */
+} __attribute__((preserve_access_index));
+
 #endif
tools/testing/selftests/bpf/progs/mptcp_bpf_rr.c new file +47 -0

47 lines not on this page · raw message

Signature · 1 line
2.34.1
#1 lore raw

PATCH v13 1/3 mptcp: add subflows array in sched data

In reply to Geliang Tang

This patch adds a subflow pointers array in struct mptcp_sched_data. Set the array before invoking get_subflow(), then get it in get_subflow() in the BPF contexts.

---
 include/net/mptcp.h                             |  2 ++
 net/mptcp/sched.c                               | 14 ++++++++++++++
 tools/testing/selftests/bpf/bpf_mptcp_helpers.h |  2 ++
 3 files changed, 18 insertions(+)
include/net/mptcp.h +2 -0
index b596ba7a8494..345f27a68eaa 100644
@@ -96,10 +96,12 @@ struct mptcp_out_options {
 };
 
 #define MPTCP_SCHED_NAME_MAX	16
+#define MPTCP_SUBFLOWS_MAX	8
 
 struct mptcp_sched_data {
 	struct sock	*sock;
 	bool		call_again;
+	struct mptcp_subflow_context *contexts[MPTCP_SUBFLOWS_MAX];
 };
 
 struct mptcp_sched_ops {
net/mptcp/sched.c +14 -0
index 3ceb721e6489..9ee2d30a6f19 100644
@@ -91,9 +91,23 @@ void mptcp_release_sched(struct mptcp_sock *msk)
 static int mptcp_sched_data_init(struct mptcp_sock *msk,
 				 struct mptcp_sched_data *data)
 {
+	struct mptcp_subflow_context *subflow;
+	int i = 0;
+
 	data->sock = NULL;
 	data->call_again = 0;
 
+	mptcp_for_each_subflow(msk, subflow) {
+		if (i == MPTCP_SUBFLOWS_MAX) {
+			pr_warn_once("too many subflows");
+			break;
+		}
+		data->contexts[i++] = subflow;
+	}
+
+	for (; i < MPTCP_SUBFLOWS_MAX; i++)
+		data->contexts[i++] = NULL;
+
 	return 0;
 }
 
tools/testing/selftests/bpf/bpf_mptcp_helpers.h +2 -0
index 4b7812c335fc..7ecd0b14666a 100644
@@ -7,10 +7,12 @@
 #include "bpf_tcp_helpers.h"
 
 #define MPTCP_SCHED_NAME_MAX	16
+#define MPTCP_SUBFLOWS_MAX	8
 
 struct mptcp_sched_data {
 	struct sock	*sock;
 	bool		call_again;
+	struct mptcp_subflow_context *contexts[MPTCP_SUBFLOWS_MAX];
 };
 
 struct mptcp_sched_ops {
Signature · 1 line
2.34.1
#3 lore raw

PATCH v13 3/3 selftests/bpf: add bpf_rr test

In reply to Geliang Tang

This patch adds the round-robin BPF MPTCP scheduler test. Use sysctl to set net.mptcp.scheduler to use this sched. Add a veth net device to simulate the multiple addresses case. Use 'ip mptcp endpoint' command to add this new endpoint to PM netlink.

---
 .../testing/selftests/bpf/prog_tests/mptcp.c  | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
tools/testing/selftests/bpf/prog_tests/mptcp.c +38 -0
index cbd9367c1e0f..5058daf15ce5 100644
@@ -6,6 +6,7 @@
 #include "cgroup_helpers.h"
 #include "network_helpers.h"
 #include "mptcp_bpf_first.skel.h"
+#include "mptcp_bpf_rr.skel.h"
 
 #ifndef TCP_CA_NAME_MAX
 #define TCP_CA_NAME_MAX	16
@@ -365,10 +366,47 @@ static void test_first(void)
 	mptcp_bpf_first__destroy(first_skel);
 }
 
+static void test_rr(void)
+{
+	struct mptcp_bpf_rr *rr_skel;
+	int server_fd, client_fd;
+	struct bpf_link *link;
+
+	rr_skel = mptcp_bpf_rr__open_and_load();
+	if (!ASSERT_OK_PTR(rr_skel, "bpf_rr__open_and_load"))
+		return;
+
+	link = bpf_map__attach_struct_ops(rr_skel->maps.rr);
+	if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) {
+		mptcp_bpf_rr__destroy(rr_skel);
+		return;
+	}
+
+	system("ip link add veth1 type veth");
+	system("ip addr add 10.0.1.1/24 dev veth1");
+	system("ip link set veth1 up");
+	system("ip mptcp endpoint add 10.0.1.1 subflow");
+	system("sysctl -qw net.mptcp.scheduler=bpf_rr");
+	server_fd = start_mptcp_server(AF_INET, NULL, 0, 0);
+	client_fd = connect_to_mptcp_fd(server_fd, 0);
+
+	send_data(server_fd, client_fd);
+
+	close(client_fd);
+	close(server_fd);
+	system("sysctl -qw net.mptcp.scheduler=default");
+	system("ip mptcp endpoint flush");
+	system("ip link del veth1");
+	bpf_link__destroy(link);
+	mptcp_bpf_rr__destroy(rr_skel);
+}
+
 void test_mptcp(void)
 {
 	if (test__start_subtest("base"))
 		test_base();
 	if (test__start_subtest("first"))
 		test_first();
+	if (test__start_subtest("rr"))
+		test_rr();
 }
Signature · 1 line
2.34.1
#4 lore raw

Re: selftests/bpf: add bpf_rr test: Tests Results

Hi Geliang,

Thank you for your modifications, that's great!

Our CI did some validations and here is its report:

- KVM Validation: normal:
- Success! ✅:
- Task: https://cirrus-ci.com/task/6702307382394880
- Summary: https://api.cirrus-ci.com/v1/artifact/task/6702307382394880/summary/summary.txt

- KVM Validation: debug:
- Unstable: 1 failed test(s): selftest_diag 🔴:
- Task: https://cirrus-ci.com/task/4556060684976128
- Summary: https://api.cirrus-ci.com/v1/artifact/task/4556060684976128/summary/summary.txt

Initiator: Patchew Applier
Commits: https://github.com/multipath-tcp/mptcp_net-next/commits/4c681daf367a

If there are some issues, you can reproduce them using the same environment as the one used by the CI thanks to a docker image, e.g.:

    $ cd [kernel source code]
    $ docker run -v "${PWD}:${PWD}:rw" -w "${PWD}" --privileged --rm -it \
        --pull always mptcp/mptcp-upstream-virtme-docker:latest \
        auto-debug

For more details:

https://github.com/multipath-tcp/mptcp-upstream-virtme-docker

Please note that despite all the efforts that have been already done to have a stable tests suite when executed on a public CI like here, it is possible some reported issues are not due to your modifications. Still, do not hesitate to help us improve that ;-)

Cheers,
MPTCP GH Action bot
Bot operated by Matthieu Baerts (Tessares)

#5 lore raw
In reply to Geliang Tang
Quoted 5 lines · Geliang Tang

v13:
- add !msk->last_snd check in patch 2
- use ASSERT_OK_PTR instead of CHECK in patch 3
- base-commit: export/20220509T115202

Thanks for the updates. Like the v12 review, I think we should wait until this is rebased on the bpf header changes before applying the patches.

- Mat

Quoted 66 lines

v12:
- init ssk from data->contexts[0], instead of msk->first.
- cycle through all the subflows, instead of the first two.

v11:
- rename array to contexts.
- drop number of subflows in mptcp_sched_data.
- set unused array elements to NULL.
- add MPTCP_SUBFLOWS_MAX check in mptcp_sched_data_init.

v10:
- init subflows array in mptcp_sched_data_init.
- for (int i = 0; i < data->subflows; i++) is not allowed in BPF, using this instead:

	for (int i = 0; i < MPTCP_SUBFLOWS_MAX && i < data->subflows; i++)

- deponds on: "BPF packet scheduler" series v18.

v9:
- add subflows array in mptcp_sched_data
- deponds on: "BPF packet scheduler" series v17 + Squash to "mptcp: add struct mptcp_sched_ops v17".

v8:
- use struct mptcp_sched_data.
- deponds on: "BPF packet scheduler" series v14.

v7:
- rename retrans to reinject.
- drop last_snd setting.
- deponds on: "BPF packet scheduler" series v13.

v6:
- set call_me_again flag.
- deponds on: "BPF packet scheduler" series v12.

v5:
- update patch 2, use temporary storage instead.
- update patch 3, use new helpers.
- deponds on: "BPF packet scheduler" series v11.

v4:
- add retrans argment for get_subflow()

v3:
- add last_snd write access.
- keep msk->last_snd setting in get_subflow().
- deponds on: "BPF packet scheduler" series v10.

v2:
- merge the squash-to patch.
- implement bpf_mptcp_get_subflows helper, instead of bpf_mptcp_get_next_subflow.
- deponds on: "BPF packet scheduler v9".

This patchset implements round-robin scheduler using BPF. Address to some commends for the RFC version:

https://patchwork.kernel.org/project/mptcp/cover/[email protected]/

Geliang Tang (3):
mptcp: add subflows array in sched data
selftests/bpf: add bpf_rr scheduler
selftests/bpf: add bpf_rr test

Quoted 13 lines
include/net/mptcp.h                           |  2 +
net/mptcp/sched.c                             | 14 ++++++
.../testing/selftests/bpf/bpf_mptcp_helpers.h |  8 ++++
.../testing/selftests/bpf/prog_tests/mptcp.c  | 38 +++++++++++++++
.../selftests/bpf/progs/mptcp_bpf_rr.c        | 47 +++++++++++++++++++
5 files changed, 109 insertions(+)

create mode 100644 tools/testing/selftests/bpf/progs/mptcp_bpf_rr.c

Signature · 1 line
2.34.1

--
Mat Martineau
Intel