Skip to contentSkip to conversation

src: mnl: fix --echo buffer size -- again

PATCH In review Florian Westphal 3 participants 6 messages Sep 9, 2019 – Sep 10, 2019 netfilter

lore
Share & tools
lore

Florian Westphal

author maintainer
#0 lore raw

PATCH 1/1 src: mnl: fix --echo buffer size -- again

Eric Garver reports:
If this restart is triggered it causes rules to be duplicated. We send the same batch again.

... and indeed, if the batch isn't doing a full replace, we cannot resend.

Therefore, remove the restart logic again.

1. If user passed --echo, use a 4mb buffer.
2. assume each element in the batch will result in a 1k notification and further increase limits if thats not enough.

This still passes on s390x (the platform that did not work with the former, more conservative estimate).

Next option (aside from increasing the guess again ...) is to add a commandline switch to nftables to allow userspace to override the buffer size.

Fixes 877baf9538f66f8f238 ("src: mnl: retry when we hit -ENOBUFS")
---
 src/mnl.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)
src/mnl.c +10 -11
index 9c1f5356c9b9..d664564e16af 100644
@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 	int ret, fd = mnl_socket_get_fd(nl), portid = mnl_socket_get_portid(nl);
 	uint32_t iov_len = nftnl_batch_iovec_len(ctx->batch);
 	char rcv_buf[MNL_SOCKET_BUFFER_SIZE];
-	unsigned int enobuf_restarts = 0;
-	size_t avg_msg_size, batch_size;
 	const struct sockaddr_nl snl = {
 		.nl_family = AF_NETLINK
 	};
@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		.tv_usec	= 0
 	};
 	struct iovec iov[iov_len];
-	unsigned int scale = 4;
 	struct msghdr msg = {};
 	fd_set readfds;
 
 	mnl_set_sndbuffer(ctx->nft->nf_sock, ctx->batch);
 
-	batch_size = mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
-	avg_msg_size = div_round_up(batch_size, num_cmds);
+	mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
 
-restart:
-	mnl_set_rcvbuffer(ctx->nft->nf_sock, num_cmds * avg_msg_size * scale);
+	if (nft_output_echo(&ctx->nft->output)) {
+		size_t buffer_size = MNL_SOCKET_BUFFER_SIZE * 1024;
+		size_t new_buffer_size = num_cmds * 1024;
+
+		if (new_buffer_size > buffer_size)
+			buffer_size = new_buffer_size;
+
+		mnl_set_rcvbuffer(ctx->nft->nf_sock, buffer_size);
+	}
 
 	ret = mnl_nft_socket_sendmsg(ctx, &msg);
 	if (ret == -1)
@@ -351,10 +354,6 @@ restart:
 
 		ret = mnl_socket_recvfrom(nl, rcv_buf, sizeof(rcv_buf));
 		if (ret == -1) {
-			if (errno == ENOBUFS && enobuf_restarts++ < 3) {
-				scale *= 2;
-				goto restart;
-			}
 			return -1;
 		}
 
Signature · 1 line
2.21.0
#1 lore raw

On Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]

Quoted 22 lines
src/mnl.c +0 -3
index 9c1f5356c9b9..d664564e16af 100644
@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 	int ret, fd = mnl_socket_get_fd(nl), portid = mnl_socket_get_portid(nl);
 	uint32_t iov_len = nftnl_batch_iovec_len(ctx->batch);
 	char rcv_buf[MNL_SOCKET_BUFFER_SIZE];
-	unsigned int enobuf_restarts = 0;
-	size_t avg_msg_size, batch_size;
 	const struct sockaddr_nl snl = {
 		.nl_family = AF_NETLINK
 	};
@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		.tv_usec	= 0
 	};
 	struct iovec iov[iov_len];
-	unsigned int scale = 4;
 	struct msghdr msg = {};
 	fd_set readfds;
 
 	mnl_set_sndbuffer(ctx->nft->nf_sock, ctx->batch);

src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331

Quoted 10 lines
src/mnl.c +4 -4
 
-	batch_size = mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
-	avg_msg_size = div_round_up(batch_size, num_cmds);
+	mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
 
-restart:
-	mnl_set_rcvbuffer(ctx->nft->nf_sock, num_cmds * avg_msg_size * scale);
+	if (nft_output_echo(&ctx->nft->output)) {
+		size_t buffer_size = MNL_SOCKET_BUFFER_SIZE * 1024;
+		size_t new_buffer_size = num_cmds * 1024;

Probably all simplify this to?

		mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);

Upper limit for us is NLMSG_GOODSIZE, which is not exposed to userspace, although we have control over that upper limit from nf_tables. In practise, I think we go over the 1 Kbytes per message boundary.

Quoted 9 lines
src/mnl.c +6 -0
+
+		if (new_buffer_size > buffer_size)
+			buffer_size = new_buffer_size;
+
+		mnl_set_rcvbuffer(ctx->nft->nf_sock, buffer_size);
+	}
 
 	ret = mnl_nft_socket_sendmsg(ctx, &msg);
 	if (ret == -1)
author maintainer
#2 lore raw
Quoted 37 lines · Pablo Neira Ayuso

On Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]

Quoted 22 lines
src/mnl.c +0 -3
index 9c1f5356c9b9..d664564e16af 100644
@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 	int ret, fd = mnl_socket_get_fd(nl), portid = mnl_socket_get_portid(nl);
 	uint32_t iov_len = nftnl_batch_iovec_len(ctx->batch);
 	char rcv_buf[MNL_SOCKET_BUFFER_SIZE];
-	unsigned int enobuf_restarts = 0;
-	size_t avg_msg_size, batch_size;
 	const struct sockaddr_nl snl = {
 		.nl_family = AF_NETLINK
 	};
@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		.tv_usec	= 0
 	};
 	struct iovec iov[iov_len];
-	unsigned int scale = 4;
 	struct msghdr msg = {};
 	fd_set readfds;
 
 	mnl_set_sndbuffer(ctx->nft->nf_sock, ctx->batch);

src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331

Quoted 10 lines
src/mnl.c +4 -4
 
-	batch_size = mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
-	avg_msg_size = div_round_up(batch_size, num_cmds);
+	mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
 
-restart:
-	mnl_set_rcvbuffer(ctx->nft->nf_sock, num_cmds * avg_msg_size * scale);
+	if (nft_output_echo(&ctx->nft->output)) {
+		size_t buffer_size = MNL_SOCKET_BUFFER_SIZE * 1024;
+		size_t new_buffer_size = num_cmds * 1024;

Probably all simplify this to?

Quoted 1 line
		mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);

Reason for above patch was to avoid any risk for normal operations by restricting the recvbuffer tuning to echo-mode and also adding a lower thresh.

For some reason I don't like the idea of setting only 1k recvbuf by default in the extreme case.

That said, it does seem to work.

#3 lore raw
Quoted 38 lines · Florian Westphal
Quoted 37 lines · Pablo Neira Ayuso

On Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]

Quoted 22 lines
src/mnl.c +0 -3
index 9c1f5356c9b9..d664564e16af 100644
@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 	int ret, fd = mnl_socket_get_fd(nl), portid = mnl_socket_get_portid(nl);
 	uint32_t iov_len = nftnl_batch_iovec_len(ctx->batch);
 	char rcv_buf[MNL_SOCKET_BUFFER_SIZE];
-	unsigned int enobuf_restarts = 0;
-	size_t avg_msg_size, batch_size;
 	const struct sockaddr_nl snl = {
 		.nl_family = AF_NETLINK
 	};
@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		.tv_usec	= 0
 	};
 	struct iovec iov[iov_len];
-	unsigned int scale = 4;
 	struct msghdr msg = {};
 	fd_set readfds;
 
 	mnl_set_sndbuffer(ctx->nft->nf_sock, ctx->batch);

src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331

Quoted 10 lines
src/mnl.c +4 -4
 
-	batch_size = mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
-	avg_msg_size = div_round_up(batch_size, num_cmds);
+	mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
 
-restart:
-	mnl_set_rcvbuffer(ctx->nft->nf_sock, num_cmds * avg_msg_size * scale);
+	if (nft_output_echo(&ctx->nft->output)) {
+		size_t buffer_size = MNL_SOCKET_BUFFER_SIZE * 1024;
+		size_t new_buffer_size = num_cmds * 1024;

Probably all simplify this to?

Quoted 8 lines
Quoted 1 line
		mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);

Reason for above patch was to avoid any risk for normal operations by restricting the recvbuffer tuning to echo-mode and also adding a lower thresh.

For some reason I don't like the idea of setting only 1k recvbuf by default in the extreme case.

I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.

Probably this? it's based on your patch.

src/mnl.c +14 -12
index 9c1f5356c9b9..8031bd6add80 100644
@@ -304,6 +304,8 @@ static ssize_t mnl_nft_socket_sendmsg(struct netlink_ctx *ctx,
 	return sendmsg(mnl_socket_get_fd(ctx->nft->nf_sock), msg, 0);
 }
 
+#define NFT_MNL_ECHO_RCVBUFF_DEFAULT	(MNL_SOCKET_BUFFER_SIZE * 1024)
+
 int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		   uint32_t num_cmds)
 {
@@ -311,8 +313,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 	int ret, fd = mnl_socket_get_fd(nl), portid = mnl_socket_get_portid(nl);
 	uint32_t iov_len = nftnl_batch_iovec_len(ctx->batch);
 	char rcv_buf[MNL_SOCKET_BUFFER_SIZE];
-	unsigned int enobuf_restarts = 0;
-	size_t avg_msg_size, batch_size;
 	const struct sockaddr_nl snl = {
 		.nl_family = AF_NETLINK
 	};
@@ -321,17 +321,24 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,
 		.tv_usec	= 0
 	};
 	struct iovec iov[iov_len];
-	unsigned int scale = 4;
 	struct msghdr msg = {};
+	unsigned int rcvbufsiz;
+	size_t batch_size;
 	fd_set readfds;
 
 	mnl_set_sndbuffer(ctx->nft->nf_sock, ctx->batch);
 
 	batch_size = mnl_nft_batch_to_msg(ctx, &msg, &snl, iov, iov_len);
-	avg_msg_size = div_round_up(batch_size, num_cmds);
 
-restart:
-	mnl_set_rcvbuffer(ctx->nft->nf_sock, num_cmds * avg_msg_size * scale);
+	if (nft_output_echo(&ctx->nft->output)) {
+		rcvbufsiz = num_cmds * 1024;
+		if (rcvbufsiz < NFT_MNL_ECHO_RCVBUFF_DEFAULT)
+			rcvbufsiz = NFT_MNL_ECHO_RCVBUFF_DEFAULT;
+	} else {
+		rcvbufsiz = num_cmds * div_round_up(batch_size, num_cmds) * 4;
+	}
+
+	mnl_set_rcvbuffer(ctx->nft->nf_sock, rcvbufsiz);
 
 	ret = mnl_nft_socket_sendmsg(ctx, &msg);
 	if (ret == -1)
@@ -350,13 +357,8 @@ restart:
 			break;
 
 		ret = mnl_socket_recvfrom(nl, rcv_buf, sizeof(rcv_buf));
-		if (ret == -1) {
-			if (errno == ENOBUFS && enobuf_restarts++ < 3) {
-				scale *= 2;
-				goto restart;
-			}
+		if (ret == -1)
 			return -1;
-		}
 
 		ret = mnl_cb_run(rcv_buf, ret, 0, portid, &netlink_echo_callback, ctx);
 		/* Continue on error, make sure we get all acknowledgments */
author maintainer
#4 lore raw
Quoted 5 lines · Pablo Neira Ayuso

I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.

Probably this? it's based on your patch.

LGTM, feel free to apply this.

Tested-by
#5 lore raw
Quoted 8 lines · Florian Westphal
Quoted 5 lines · Pablo Neira Ayuso

I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.

Probably this? it's based on your patch.

LGTM, feel free to apply this.

Pablo's version passes all my tests. But so does Florian's version. Feel free to add my tested-by tag.