Skip to contentSkip to conversation

selftests: nft_queue.sh: add a parallel stress test

PATCH In review Fernando Fernandez Mancera 2 participants 3 messages Apr 6 – Apr 7

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 selftests: nft_queue.sh: add a parallel stress test

Introduce a new stress test to check for race conditions in the nfnetlink_queue subsystem, where an entry is freed while another CPU is concurrently walking the global rhashtable.

To trigger this, nf_queue.c is extended with two new flags:
* -O (out-of-order): Buffers packet IDs and flushes them in reverse.
* -b (bogus verdicts): Floods the kernel with non-existent packet IDs.

The bogus verdict loop forces the kernel's lookup function to perform full rhashtable bucket traversals (-ENOENT). Combined with reverse-order flushing and heavy parallel UDP/ping flooding across 8 queues, this puts the nfnetlink_queue code under pressure.

Joint work with Florian Westphal.

---
 .../selftests/net/netfilter/nf_queue.c        | 50 +++++++++--
 .../selftests/net/netfilter/nft_queue.sh      | 83 ++++++++++++++++---
 2 files changed, 115 insertions(+), 18 deletions(-)
tools/testing/selftests/net/netfilter/nf_queue.c +44 -6
index 116c0ca0eabb..8bbec37f5356 100644
@@ -19,6 +19,8 @@ struct options {
 	bool count_packets;
 	bool gso_enabled;
 	bool failopen;
+	bool out_of_order;
+	bool bogus_verdict;
 	int verbose;
 	unsigned int queue_num;
 	unsigned int timeout;
@@ -31,7 +33,7 @@ static struct options opts;
 
 static void help(const char *p)
 {
-	printf("Usage: %s [-c|-v [-vv] ] [-o] [-t timeout] [-q queue_num] [-Qdst_queue ] [ -d ms_delay ] [-G]\n", p);
+	printf("Usage: %s [-c|-v [-vv] ] [-o] [-O] [-b] [-t timeout] [-q queue_num] [-Qdst_queue ] [ -d ms_delay ] [-G]\n", p);
 }
 
 static int parse_attr_cb(const struct nlattr *attr, void *data)
@@ -275,7 +277,9 @@ static int mainloop(void)
 	unsigned int buflen = 64 * 1024 + MNL_SOCKET_BUFFER_SIZE;
 	struct mnl_socket *nl;
 	struct nlmsghdr *nlh;
+	uint32_t ooo_ids[16];
 	unsigned int portid;
+	int ooo_count = 0;
 	char *buf;
 	int ret;
 
@@ -308,6 +312,9 @@ static int mainloop(void)
 
 		ret = mnl_cb_run(buf, ret, 0, portid, queue_cb, NULL);
 		if (ret < 0) {
+			/* bogus verdict mode will generate ENOENT error messages */
+			if (opts.bogus_verdict && errno == ENOENT)
+				continue;
 			perror("mnl_cb_run");
 			exit(EXIT_FAILURE);
 		}
@@ -316,10 +323,35 @@ static int mainloop(void)
 		if (opts.delay_ms)
 			sleep_ms(opts.delay_ms);
 
-		nlh = nfq_build_verdict(buf, id, opts.queue_num, opts.verdict);
-		if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) {
-			perror("mnl_socket_sendto");
-			exit(EXIT_FAILURE);
+		if (opts.bogus_verdict) {
+			for (int i = 0; i < 50; i++) {
+				nlh = nfq_build_verdict(buf, id + 0x7FFFFFFF + i,
+							opts.queue_num, opts.verdict);
+				mnl_socket_sendto(nl, nlh, nlh->nlmsg_len);
+			}
+		}
+
+		if (opts.out_of_order) {
+			ooo_ids[ooo_count] = id;
+			if (ooo_count >= 15) {
+				for (ooo_count; ooo_count >= 0; ooo_count--) {
+					nlh = nfq_build_verdict(buf, ooo_ids[ooo_count],
+								opts.queue_num, opts.verdict);
+					if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) {
+						perror("mnl_socket_sendto");
+						exit(EXIT_FAILURE);
+					}
+				}
+				ooo_count = 0;
+			} else {
+				ooo_count++;
+			}
+		} else {
+			nlh = nfq_build_verdict(buf, id, opts.queue_num, opts.verdict);
+			if (mnl_socket_sendto(nl, nlh, nlh->nlmsg_len) < 0) {
+				perror("mnl_socket_sendto");
+				exit(EXIT_FAILURE);
+			}
 		}
 	}
 
@@ -332,7 +364,7 @@ static void parse_opts(int argc, char **argv)
 {
 	int c;
 
-	while ((c = getopt(argc, argv, "chvot:q:Q:d:G")) != -1) {
+	while ((c = getopt(argc, argv, "chvoObt:q:Q:d:G")) != -1) {
 		switch (c) {
 		case 'c':
 			opts.count_packets = true;
@@ -375,6 +407,12 @@ static void parse_opts(int argc, char **argv)
 		case 'v':
 			opts.verbose++;
 			break;
+		case 'O':
+			opts.out_of_order = true;
+			break;
+		case 'b':
+			opts.bogus_verdict = true;
+			break;
 		}
 	}
 
tools/testing/selftests/net/netfilter/nft_queue.sh +71 -12

107 lines not on this page · raw message

Signature · 1 line
2.53.0
maintainer
#1 lore raw

tools/testing/selftests/net/netfilter/nft_queue.sh ret=0 · line 14

Quoted 9 lines · Fernando Fernandez Mancera
tools/testing/selftests/net/netfilter/nft_queue.sh +1 -0
index ea766bdc5d04..1e1949c6a918 100755
@@ -11,6 +11,7 @@ ret=0
 timeout=5
 
 SCTP_TEST_TIMEOUT=60
+STRESS_TEST_TIMEOUT=300

I changed this to 30s, I think the full 5m dance is a bit too long for the selftest.

I know 30s is likely not enough to trigger the bug reliably but given this test will run may times per day on netdev infra I think 30s is ok.

#2 lore raw
Quoted 8 lines · Florian Westphal
Quoted 7 lines · Fernando Fernandez Mancera
tools/testing/selftests/net/netfilter/nft_queue.sh +0 -0
index ea766bdc5d04..1e1949c6a918 100755
@@ -11,6 +11,7 @@ ret=0
  timeout=5
 
Quoted 8 lines

tools/testing/selftests/net/netfilter/nft_queue.sh ret=0 · line 14

Quoted 2 lines
tools/testing/selftests/net/netfilter/nft_queue.sh +1 -0
  SCTP_TEST_TIMEOUT=60
+STRESS_TEST_TIMEOUT=300

I changed this to 30s, I think the full 5m dance is a bit too long for the selftest.

I know 30s is likely not enough to trigger the bug reliably but given this test will run may times per day on netdev infra I think 30s is ok.

It makes sense. Indeed, it is enough time. At least as tested on my system.