mnl: add mnl_set_rcvbuffer() and use it
Pablo Neira Ayuso
maintainerPATCH v2 1/7 mnl: add mnl_set_rcvbuffer() and use it
This new function allows us to set the netlink receiver buffer.
--- src/mnl.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-)
src/mnl.c +23 -14
@@ -233,6 +233,23 @@ static void mnl_set_sndbuffer(const struct mnl_socket *nl, nlbuffsiz = newbuffsiz; } +static int mnl_set_rcvbuffer (const struct mnl_socket *nl,size_t bufsiz) +{ +int ret; + + ret =setsockopt (mnl_socket_get_fd (nl),SOL_SOCKET ,SO_RCVBUFFORCE , + &bufsiz,sizeof (socklen_t)); +if (ret <0 ) { +/* If this doesn't work, try to reach the system wide maximum +* (or whatever the user requested). +*/ + ret =setsockopt (mnl_socket_get_fd (nl),SOL_SOCKET ,SO_RCVBUF , + &bufsiz,sizeof (socklen_t)); + } + +return ret; +} +static ssize_t mnl_nft_socket_sendmsg (const struct netlink_ctx *ctx) {static const struct sockaddr_nl snl = {
@@ -1391,20 +1408,12 @@ int mnl_nft_event_listener(struct mnl_socket *nf_sock, unsigned int debug_mask,fd_set readfds;int ret; - ret =setsockopt (fd,SOL_SOCKET ,SO_RCVBUFFORCE , &bufsiz, -sizeof (socklen_t)); -if (ret <0 ) { -/* If this doesn't work, try to reach the system wide maximum -* (or whatever the user requested). -*/ - ret =setsockopt (fd,SOL_SOCKET ,SO_RCVBUF , &bufsiz, -sizeof (socklen_t)); -if (ret <0 ) -nft_print (octx,"# Cannot increase netlink socket buffer size, expect message loss\n" ); -else -nft_print (octx,"# Cannot set up netlink socket buffer size to %u bytes, falling back to %u bytes\n" , -NFTABLES_NLEVENT_BUFSIZ , bufsiz); - } + ret =mnl_set_rcvbuffer (nf_sock, bufsiz); +if (ret <0 ) +nft_print (octx,"# Cannot increase netlink socket buffer size, expect message loss\n" ); +else +nft_print (octx,"# Cannot set up netlink socket buffer size to %u bytes, falling back to %u bytes\n" , +NFTABLES_NLEVENT_BUFSIZ , bufsiz);while (1 ) {FD_ZERO (&readfds);
Signature · 1 line
2.11.0
Pablo Neira Ayuso
maintainerPATCH v2 2/7 mnl: mnl_set_rcvbuffer() skips buffer size update if it is too small
Check for existing buffer size, if this is larger than the requested new buffer size, skip the buffer size update.
--- src/mnl.c | 7 +++++++ 1 file changed, 7 insertions(+)
src/mnl.c +7 -0
@@ -235,8 +235,15 @@ static void mnl_set_sndbuffer(const struct mnl_socket *nl,static int mnl_set_rcvbuffer (const struct mnl_socket *nl,size_t bufsiz) { +size_t cur_bufsiz; +socklen_t len;int ret; + ret =getsockopt (mnl_socket_get_fd (nl),SOL_SOCKET ,SO_RCVBUF , + &cur_bufsiz, &len); +if (cur_bufsiz > bufsiz) +return 0 ; + ret =setsockopt (mnl_socket_get_fd (nl),SOL_SOCKET ,SO_RCVBUFFORCE , &bufsiz,sizeof (socklen_t));if (ret <0 ) {
Signature · 1 line
2.11.0
Pablo Neira Ayuso
maintainerPATCH v2 3/7 mnl: call mnl_set_sndbuffer() from mnl_batch_talk()
In reply to Pablo Neira AyusoInstead of mnl_nft_socket_sendmsg(), just a cleanup.
--- src/mnl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
src/mnl.c +2 -1
@@ -272,7 +272,6 @@ static ssize_t mnl_nft_socket_sendmsg(const struct netlink_ctx *ctx) };uint32_t i; -mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);nftnl_batch_iovec (ctx->batch, iov, iov_len);for (i =0 ; i < iov_len; i++) {
@@ -298,6 +297,8 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) };int err =0 ; +mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); + ret =mnl_nft_socket_sendmsg (ctx);if (ret ==-1 )return -1 ;
Signature · 1 line
2.11.0
Pablo Neira Ayuso
maintainerPATCH v2 4/7 mnl: add mnl_nft_batch_to_msg()
In reply to Pablo Neira AyusoThis function transforms the batch into a msghdr object.
--- src/mnl.c | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-)
src/mnl.c +36 -18
@@ -257,49 +257,67 @@ static int mnl_set_rcvbuffer(const struct mnl_socket *nl, size_t bufsiz)return ret; } -static ssize_t mnl_nft_socket_sendmsg (const struct netlink_ctx *ctx) +static size_t mnl_nft_batch_to_msg (struct netlink_ctx *ctx,struct msghdr *msg, +const struct sockaddr_nl *snl, +struct iovec *iov,unsigned int iov_len) { -static const struct sockaddr_nl snl = { - .nl_family =AF_NETLINK - }; -uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch); -struct iovec iov[iov_len]; -struct msghdr msg = { - .msg_name = (struct sockaddr *) &snl, - .msg_namelen =sizeof (snl), - .msg_iov = iov, - .msg_iovlen = iov_len, - }; -uint32_t i; +unsigned int i; +size_t len =0 ; + + msg->msg_name = (struct sockaddr_nl *)snl; + msg->msg_namelen =sizeof (*snl); + msg->msg_iov = iov; + msg->msg_iovlen = iov_len;nftnl_batch_iovec (ctx->batch, iov, iov_len); -for (i =0 ; i < iov_len; i++) { -if (ctx->nft->debug_mask &NFT_DEBUG_MNL ) { +for (i =0 ; i < iov_len; i++) + len += msg->msg_iov[i].iov_len; + +return len; +} + +static ssize_t mnl_nft_socket_sendmsg (struct netlink_ctx *ctx, +const struct msghdr *msg) +{ +uint32_t iov_len = msg->msg_iovlen; +struct iovec *iov = msg->msg_iov; +unsigned int i; + +if (ctx->nft->debug_mask &NFT_DEBUG_MNL ) { +for (i =0 ; i < iov_len; i++) {mnl_nlmsg_fprintf (ctx->nft->output.output_fp, iov[i].iov_base, iov[i].iov_len,sizeof (struct nfgenmsg )); } } -return sendmsg (mnl_socket_get_fd (ctx->nft->nf_sock), &msg,0 ); +return sendmsg (mnl_socket_get_fd (ctx->nft->nf_sock), msg,0 ); }int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list) {struct mnl_socket *nl = ctx->nft->nf_sock;int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl); +uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; -fd_set readfds; +const struct sockaddr_nl snl = { + .nl_family =AF_NETLINK + };struct timeval tv = { .tv_sec =0 , .tv_usec =0 }; +fd_set readfds; +struct iovec iov[iov_len]; +struct msghdr msg = {};int err =0 ;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); - ret =mnl_nft_socket_sendmsg (ctx); +mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + + ret =mnl_nft_socket_sendmsg (ctx, &msg);if (ret ==-1 )return -1 ;
Signature · 1 line
2.11.0
Pablo Neira Ayuso
maintainerPATCH v2 5/7 mnl: estimate receiver buffer size
In reply to Pablo Neira AyusoSet a receiver buffer size based on the number of commands and the average message size, this is useful for the --echo option in order to avoid ENOBUFS errors.
Double the estimated size is used to ensure enough receiver buffer space.
Skip buffer receiver logic if estimation is smaller than current buffer.
--- include/mnl.h | 3 ++- src/libnftables.c | 5 +++-- src/mnl.c | 11 ++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-)
include/mnl.h +2 -1
@@ -25,7 +25,8 @@ bool mnl_batch_ready(struct nftnl_batch *batch);void mnl_batch_reset (struct nftnl_batch *batch);uint32_t mnl_batch_begin (struct nftnl_batch *batch,uint32_t seqnum);void mnl_batch_end (struct nftnl_batch *batch,uint32_t seqnum); -int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list); +int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list, +uint32_t num_cmds);int mnl_nft_rule_add (struct netlink_ctx *ctx,const struct cmd *cmd,unsigned int flags);
src/libnftables.c +3 -2
@@ -21,7 +21,7 @@ static int nft_netlink(struct nft_ctx *nft, struct list_head *cmds, struct list_head *msgs, struct mnl_socket *nf_sock) { - uint32_t batch_seqnum, seqnum = 0; + uint32_t batch_seqnum, seqnum = 0, num_cmds = 0;struct nftnl_batch *batch;struct netlink_ctx ctx;struct cmd *cmd;
@@ -49,6 +49,7 @@ static int nft_netlink(struct nft_ctx *nft, strerror(errno)); goto out; } + num_cmds++; }if (!nft->check)mnl_batch_end (batch,mnl_seqnum_alloc (&seqnum));
@@ -56,7 +57,7 @@ static int nft_netlink(struct nft_ctx *nft,if (!mnl_batch_ready (batch)) goto out; - ret =mnl_batch_talk (&ctx, &err_list); + ret =mnl_batch_talk (&ctx, &err_list, num_cmds);list_for_each_entry_safe (err, tmp, &err_list, head) {list_for_each_entry (cmd, cmds, list) {
src/mnl.c +8 -3
@@ -295,12 +295,14 @@ static ssize_t mnl_nft_socket_sendmsg(struct netlink_ctx *ctx,return sendmsg (mnl_socket_get_fd (ctx->nft->nf_sock), msg,0 ); } -int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list) +int mnl_batch_talk (struct netlink_ctx *ctx,struct list_head *err_list, +uint32_t num_cmds) {struct mnl_socket *nl = ctx->nft->nf_sock;int ret, fd =mnl_socket_get_fd (nl), portid =mnl_socket_get_portid (nl);uint32_t iov_len =nftnl_batch_iovec_len (ctx->batch);char rcv_buf[MNL_SOCKET_BUFFER_SIZE ]; +size_t avg_msg_size, batch_size;const struct sockaddr_nl snl = { .nl_family =AF_NETLINK };
@@ -308,14 +310,17 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) .tv_sec = 0, .tv_usec = 0 }; -fd_set readfds;struct iovec iov[iov_len];struct msghdr msg = {}; +fd_set readfds;int err =0 ;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); -mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + avg_msg_size =div_round_up (batch_size, num_cmds); + +mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size *2 ); ret =mnl_nft_socket_sendmsg (ctx, &msg);if (ret ==-1 )
Signature · 1 line
2.11.0
Re: [PATCH nft,v2 5/7] mnl: estimate receiver buffer size
Quoted 12 lines · Pablo Neira Ayuso
Set a receiver buffer size based on the number of commands and the average message size, this is useful for the --echo option in order to avoid ENOBUFS errors.
Double the estimated size is used to ensure enough receiver buffer space.
Skip buffer receiver logic if estimation is smaller than current buffer.
Reported-by Phil Sutter <[email protected]>Signed-off-by Pablo Neira Ayuso <[email protected]>---
[..]
Quoted 4 lines
src/libnftables.c +0
-0
[..]
Quoted 7 lines
src/libnftables.c +0
-1@@ -308,14 +310,17 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) .tv_sec = 0, .tv_usec = 0 }; -fd_set readfds;struct iovec iov[iov_len];struct msghdr msg = {};
src/libnftables.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) · line 323
Quoted 10 lines
src/libnftables.c +5
-1+fd_set readfds;int err =0 ;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch); -mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + avg_msg_size =div_round_up (batch_size, num_cmds); + +mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size *2 );
I think this calculation is incorrect. I'm still getting ENOBUFS with Phil's testcase and firewalld's testsuite (large json blob). I changed the multiplier from 2 to 6 and it worked.
-->8--
# ./run-tests.sh ./testcases/transactions/0049huge_0 I: using nft binary ./../../src/nft
W: [FAILED] ./testcases/transactions/0049huge_0: got 1
netlink: Error: Could not process rule: No buffer space available
I: results: [OK] 0 [FAILED] 1 [TOTAL] 1
Pablo Neira Ayuso
maintainerRe: [PATCH nft,v2 5/7] mnl: estimate receiver buffer size
Quoted 31 lines · Eric Garver
Quoted 12 lines · Pablo Neira Ayuso
Set a receiver buffer size based on the number of commands and the average message size, this is useful for the --echo option in order to avoid ENOBUFS errors.
Double the estimated size is used to ensure enough receiver buffer space.
Skip buffer receiver logic if estimation is smaller than current buffer.
Reported-by Phil Sutter <[email protected]>Signed-off-by Pablo Neira Ayuso <[email protected]>---
[..]
Quoted 4 lines
src/libnftables.c +0
-0[..]
Quoted 12 lines
src/libnftables.c +1
-1@@ -308,14 +310,17 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) .tv_sec = 0, .tv_usec = 0 }; -fd_set readfds;struct iovec iov[iov_len];struct msghdr msg = {}; +fd_set readfds;int err =0 ;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);
Quoted 7 lines
src/libnftables.c int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list) · line 323
Quoted 5 lines
src/libnftables.c +4
-1-mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + batch_size =mnl_nft_batch_to_msg (ctx, &msg, &snl, iov, iov_len); + avg_msg_size =div_round_up (batch_size, num_cmds); + +mnl_set_rcvbuffer (ctx->nft->nf_sock, num_cmds * avg_msg_size *2 );I think this calculation is incorrect.
Yes, see v4 of this patch:
https://patchwork.ozlabs.org/patch/1107737/
Quoted 3 lines
I'm still getting ENOBUFS with Phil's testcase and firewalld's testsuite (large json blob). I changed the multiplier from 2 to 6 and it worked.
I just pushed out the patchset, the last version is using a multiplier of 4, I modified Phil's testcase to 100000 and it works fine. Please try the version upstream and let me know.
We can enhance this code by checking for ENOBUFS in _sendmsg(), extend the buffer size and retry. Then, we also need to update kernel code to abort the transaction in case NLM_F_ECHO flag is set on and we hit ENOBUFS.
Pablo Neira Ayuso
maintainerPATCH v2 6/7 mnl: mnl_batch_talk() returns -1 on internal netlink errors
In reply to Pablo Neira AyusoDisplay an error in case internal netlink plumbing hits problems.
--- src/libnftables.c | 8 ++++++++ src/mnl.c | 7 ++----- 2 files changed, 10 insertions(+), 5 deletions(-)
src/libnftables.c +8 -0
@@ -58,6 +58,14 @@ static int nft_netlink(struct nft_ctx *nft, goto out; ret =mnl_batch_talk (&ctx, &err_list, num_cmds); +if (ret <0 ) { +netlink_io_error (&ctx,NULL , +"Could not process rule: %s" ,strerror (errno)); + goto out; + } + +if (!list_empty (&err_list)) + ret =-1 ;list_for_each_entry_safe (err, tmp, &err_list, head) {list_for_each_entry (cmd, cmds, list) {
src/mnl.c +2 -5
@@ -313,7 +313,6 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,struct iovec iov[iov_len];struct msghdr msg = {};fd_set readfds; -int err =0 ;mnl_set_sndbuffer (ctx->nft->nf_sock, ctx->batch);
@@ -343,10 +342,8 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list, ret =mnl_cb_run (rcv_buf, ret,0 , portid, &netlink_echo_callback, ctx);/* Continue on error, make sure we get all acknowledgments */ -if (ret ==-1 ) { +if (ret ==-1 )mnl_err_list_node_add (err_list, errno, nlh->nlmsg_seq); - err =-1 ; - } ret =select (fd+1 , &readfds,NULL ,NULL , &tv);if (ret ==-1 )
@@ -355,7 +352,7 @@ int mnl_batch_talk(struct netlink_ctx *ctx, struct list_head *err_list,FD_ZERO (&readfds);FD_SET (fd, &readfds); } -return err; +return 0 ; } int mnl_nft_rule_add(struct netlink_ctx *ctx, const struct cmd *cmd,
Signature · 1 line
2.11.0
Pablo Neira Ayuso
maintainerPATCH v2 7/7 erec: remove double \n on error when internal_netlink is used
In reply to Pablo Neira AyusoRemove double empty line linebreak when printing internal errors.
--- src/erec.c | 1 - 1 file changed, 1 deletion(-)
src/erec.c +0 -1
@@ -161,7 +161,6 @@ void erec_print(struct output_ctx *octx, const struct error_record *erec, loc = &erec->locations[l];netlink_dump_expr (loc->nle, f, debug_mask); } -fprintf (f,"\n\n" );return ; }
Signature · 1 line
2.11.0