src: mnl: fix --echo buffer size -- again
Florian Westphal
maintainerPATCH 1/1 src: mnl: fix --echo buffer size -- again
Eric Garver reports:
If this restart is triggered it causes rules to be duplicated. We send the same batch again.
... and indeed, if the batch isn't doing a full replace, we cannot resend.
Therefore, remove the restart logic again.
1. If user passed --echo, use a 4mb buffer.
2. assume each element in the batch will result in a 1k notification and further increase limits if thats not enough.
This still passes on s390x (the platform that did not work with the former, more conservative estimate).
Next option (aside from increasing the guess again ...) is to add a commandline switch to nftables to allow userspace to override the buffer size.
--- src/mnl.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)
src/mnl.c +10 -11
@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -unsigned int enobuf_restarts =0 ; -size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };
@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, .tv_usec = 0 };struct iovec iov[iov_len]; -unsigned int scale =4 ;struct msghdr msg = {};fd_set readfds;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); - batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); - avg_msg_size =div_round_up (batch_size, num_cmds); +mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); -restart: -mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size * scale); +if (nft_output_echo (&ctx->nft->output)) { +size_t buffer_size =MNL_SOCKET_BUFFER_SIZE *1024 ; +size_t new_buffer_size = num_cmds *1024 ; + +if (new_buffer_size > buffer_size) + buffer_size = new_buffer_size; + +mnl_set_rcvbuffer (ctx->nft->nf_sock, buffer_size); + } ret =mnl_nft_socket_sendmsg (ctx, &msg);if (ret ==-1 )
@@ -351,10 +354,6 @@ restart: ret =mnl_socket_recvfrom (nl, rcv_buf,sizeof (rcv_buf));if (ret ==-1 ) { -if (errno ==ENOBUFS && enobuf_restarts++ <3 ) { - scale *=2 ; - goto restart; - }return -1 ; }
Signature · 1 line
2.21.0
Pablo Neira Ayuso
maintainerOn Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]
Quoted 22 lines
src/mnl.c +0
-3@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -unsigned int enobuf_restarts =0 ; -size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, .tv_usec = 0 };struct iovec iov[iov_len]; -unsigned int scale =4 ;struct msghdr msg = {};fd_set readfds;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);
src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331
Quoted 10 lines
src/mnl.c +4
-4- batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); - avg_msg_size =div_round_up (batch_size, num_cmds); +mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); -restart: -mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size * scale); +if (nft_output_echo (&ctx->nft->output)) { +size_t buffer_size =MNL_SOCKET_BUFFER_SIZE *1024 ; +size_t new_buffer_size = num_cmds *1024 ;
Probably all simplify this to?
mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);
Upper limit for us is NLMSG_GOODSIZE, which is not exposed to userspace, although we have control over that upper limit from nf_tables. In practise, I think we go over the 1 Kbytes per message boundary.
Quoted 9 lines
src/mnl.c +6
-0+ +if (new_buffer_size > buffer_size) + buffer_size = new_buffer_size; + +mnl_set_rcvbuffer (ctx->nft->nf_sock, buffer_size); + } ret =mnl_nft_socket_sendmsg (ctx, &msg);if (ret ==-1 )
Florian Westphal
maintainerQuoted 37 lines · Pablo Neira Ayuso
On Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]Quoted 22 lines
src/mnl.c +0
-3@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -unsigned int enobuf_restarts =0 ; -size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, .tv_usec = 0 };struct iovec iov[iov_len]; -unsigned int scale =4 ;struct msghdr msg = {};fd_set readfds;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331
Quoted 10 lines
src/mnl.c +4
-4- batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); - avg_msg_size =div_round_up (batch_size, num_cmds); +mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); -restart: -mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size * scale); +if (nft_output_echo (&ctx->nft->output)) { +size_t buffer_size =MNL_SOCKET_BUFFER_SIZE *1024 ; +size_t new_buffer_size = num_cmds *1024 ;Probably all simplify this to?
Quoted 1 line
mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);
Reason for above patch was to avoid any risk for normal operations by restricting the recvbuffer tuning to echo-mode and also adding a lower thresh.
For some reason I don't like the idea of setting only 1k recvbuf by default in the extreme case.
That said, it does seem to work.
Pablo Neira Ayuso
maintainerQuoted 38 lines · Florian Westphal
Quoted 37 lines · Pablo Neira Ayuso
On Tue, Sep 10, 2019 at 12:19:18AM +0200, Florian Westphal wrote:
[...]Quoted 22 lines
src/mnl.c +0
-3@@ -311,8 +311,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -unsigned int enobuf_restarts =0 ; -size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };@@ -321,17 +319,22 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, .tv_usec = 0 };struct iovec iov[iov_len]; -unsigned int scale =4 ;struct msghdr msg = {};fd_set readfds;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);src/mnl.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, · line 331
Quoted 10 lines
src/mnl.c +4
-4- batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); - avg_msg_size =div_round_up (batch_size, num_cmds); +mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); -restart: -mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size * scale); +if (nft_output_echo (&ctx->nft->output)) { +size_t buffer_size =MNL_SOCKET_BUFFER_SIZE *1024 ; +size_t new_buffer_size = num_cmds *1024 ;Probably all simplify this to?
Quoted 8 lines
Quoted 1 line
mnl_set_rcvbuffer(ctx->nft->nf_sock, (1 << 10) * num_cmds);Reason for above patch was to avoid any risk for normal operations by restricting the recvbuffer tuning to echo-mode and also adding a lower thresh.
For some reason I don't like the idea of setting only 1k recvbuf by default in the extreme case.
I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.
Probably this? it's based on your patch.
src/mnl.c +14 -12
@@ -304,6 +304,8 @@ static ssize_t mnl_nft_socket_sendmsg(struct netlink_ctx *ctx,return sendmsg (mnl_socket_get_fd (ctx->nft->nf_sock), msg,0 ); } +#define NFT_MNL_ECHO_RCVBUFF_DEFAULT (MNL_SOCKET_BUFFER_SIZE * 1024) +int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list,uint32_t num_cmds) {
@@ -311,8 +313,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -unsigned int enobuf_restarts =0 ; -size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };
@@ -321,17 +321,24 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, .tv_usec = 0 };struct iovec iov[iov_len]; -unsigned int scale =4 ;struct msghdr msg = {}; +unsigned int rcvbufsiz; +size_t batch_size;fd_set readfds;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); - avg_msg_size =div_round_up (batch_size, num_cmds); -restart: -mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size * scale); +if (nft_output_echo (&ctx->nft->output)) { + rcvbufsiz = num_cmds *1024 ; +if (rcvbufsiz <NFT_MNL_ECHO_RCVBUFF_DEFAULT ) + rcvbufsiz =NFT_MNL_ECHO_RCVBUFF_DEFAULT ; + }else { + rcvbufsiz = num_cmds *div_round_up (batch_size, num_cmds) *4 ; + } + +mnl_set_rcvbuffer (ctx->nft->nf_sock, rcvbufsiz); ret =mnl_nft_socket_sendmsg (ctx, &msg);if (ret ==-1 )
@@ -350,13 +357,8 @@ restart:break ; ret =mnl_socket_recvfrom (nl, rcv_buf,sizeof (rcv_buf)); -if (ret ==-1 ) { -if (errno ==ENOBUFS && enobuf_restarts++ <3 ) { - scale *=2 ; - goto restart; - } +if (ret ==-1 )return -1 ; - } ret =mnl_cb_run (rcv_buf, ret,0 , portid, &netlink_echo_callback, ctx);/* Continue on error, make sure we get all acknowledgments */
Florian Westphal
maintainerQuoted 5 lines · Pablo Neira Ayuso
I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.
Probably this? it's based on your patch.
LGTM, feel free to apply this.
Eric Garver
Tested-byQuoted 8 lines · Florian Westphal
Quoted 5 lines · Pablo Neira Ayuso
I'd still like to keep setting the receive buffer for the non-echo case, a ruleset with lots of acknowledments (lots of errors) might hit ENOBUFS, I remember that was reproducible.
Probably this? it's based on your patch.
LGTM, feel free to apply this.
Pablo's version passes all my tests. But so does Florian's version. Feel free to add my tested-by tag.