mptcp: refactor xmit path
Subsystems (2)
Paolo Abeni
v3 cover mptcp: refactor xmit path
this series refactor the mptcp xmit path trying to make sendmsg behavior more consistent, memory efficent and improve its performances.
Self-test results are not affected: tests fail in similar way before and after this series.
To support MP_JOIN and multiple subflows, the idea is adding, with later patches, an RB-tree structure containing references to the pending page fragments, their length and the associated mptcp write_seq number (is subflow rel_write_seq needed, too?). RB-tree node can be allocated using the same page frag used in this series for skb data. Each RB-tree node will own a reference to the page frag.
v2 -> v3:
- rebased on top of "mptcp: refactor shutdown/close and conn_list handling"
- cope with egres pkt with NULL ext
v1 -> v2:
- rebased on top of current git
- fix handling of some corner cases
RFC -> v1:
- addressed Mat's feedback, dropping patch 3/4 and switching to per msk page frag use, and checking mptcp seq number for coalescing
Paolo Abeni (3):
mptcp: use sk_page_frag() in sendmsg
mptcp: sendmsg() do spool all the provided data
mptcp: allow collapsing consecutive sendpages on the same substream
net/mptcp/protocol.c | 190 ++++++++++++++++++++++++++++--------------- 1 file changed, 123 insertions(+), 67 deletions(-)
Signature · 1 line
2.20.1
Paolo Abeni
PATCH v3 1/3 mptcp: use sk_page_frag() in sendmsg
This clean-up a bit the send path, and allows better performances.
RFC -> v1:
- use sk's page_frag
- fix error handling on memory pressure
Signed-off-by: Paolo Abeni <pabeni(a)redhat.com>
--
can be squashed into commit cc0d09002896 ("mptcp: Implement MPTCP receive path")
--- net/mptcp/protocol.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-)
net/mptcp/protocol.c +20 -21
@@ -53,10 +53,11 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)struct mptcp_sock *msk =mptcp_sk (sk);int mss_now, size_goal, poffset, ret;struct mptcp_ext *mpext =NULL ; -struct page *page =NULL ; +struct page_frag *pfrag;struct sk_buff *skb;struct sock *ssk;size_t psize; +long timeo;pr_debug ("msk=%p" , msk);if (msk->subflow) {
@@ -81,33 +82,33 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) goto put_out; } -/* Initial experiment: new page per send. Real code will -* maintain list of active pages and DSS mappings, append to the -* end and honor zerocopy +lock_sock (sk); +lock_sock (ssk); + timeo =sock_sndtimeo (sk, msg->msg_flags &MSG_DONTWAIT ); + +/* use the mptcp page cache so that we can easily move the data +* from one substream to another, but do per subflow memory accounting */ - page =alloc_page (GFP_KERNEL ); -if (!page) { - ret = -ENOMEM ; - goto put_out; + pfrag =sk_page_frag (sk); +while (!sk_page_frag_refill (ssk, pfrag)) { + ret =sk_stream_wait_memory (ssk, &timeo); +if (ret) + goto release_out; }/* Copy to page */ - poffset =0 ; + poffset = pfrag->offset;pr_debug ("left=%zu" ,msg_data_left (msg)); - psize =copy_page_from_iter (page, poffset, + psize =copy_page_from_iter (pfrag->page, poffset,min_t (size_t,msg_data_left (msg), -PAGE_SIZE ), + pfrag->size - poffset), &msg->msg_iter);pr_debug ("left=%zu" ,msg_data_left (msg)); -if (!psize) { ret = -EINVAL ; - goto put_out; + goto release_out; } -lock_sock (sk); -lock_sock (ssk); -/* Mark the end of the previous write so the beginning of the * next write (with its own mptcp skb extension data) is not * collapsed.
@@ -117,8 +118,8 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)TCP_SKB_CB (skb)->eor =1 ; mss_now =tcp_send_mss (ssk, &size_goal, msg->msg_flags); - - ret =do_tcp_sendpages (ssk, page, poffset,min_t (int, size_goal, psize), + psize =min_t (int, size_goal, psize); + ret =do_tcp_sendpages (ssk, pfrag->page, poffset, psize, msg->msg_flags |MSG_SENDPAGE_NOTLAST );if (ret <=0 ) goto release_out;
@@ -144,6 +145,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) mpext->checksum, mpext->dsn64); }/* TODO: else fallback */ + pfrag->offset += ret; msk->write_seq += ret;subflow_ctx (ssk)->rel_write_seq += ret;
@@ -154,9 +156,6 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)release_sock (sk); put_out: -if (page) -put_page (page); -sock_put (ssk);return ret; }
Signature · 1 line
2.20.1
Paolo Abeni
PATCH v3 2/3 mptcp: sendmsg() do spool all the provided data
In reply to Paolo AbeniThis makes mptcp sendmsg() behaviour more consistent and improves xmit performances.
v2 -> v3:
- rebased
v1 -> v2:
- be sure to not advance the iov iter beyond the size_goal
- if do_tcp_sendpages() is unable to spool al the provided data, sync accordingly the iov_iter
--- net/mptcp/protocol.c | 110 +++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 47 deletions(-)
net/mptcp/protocol.c +63 -47
@@ -48,66 +48,37 @@ static struct sock *mptcp_subflow_get_ref(const struct mptcp_sock *msk)return NULL ; } -static int mptcp_sendmsg (struct sock *sk,struct msghdr *msg,size_t len) +static int mptcp_sendmsg_frag (struct sock *sk,struct sock *ssk, +struct msghdr *msg,long *timeo) {struct mptcp_sock *msk =mptcp_sk (sk); -int mss_now, size_goal, poffset, ret;struct mptcp_ext *mpext =NULL ; +int mss_now, size_goal, ret;struct page_frag *pfrag;struct sk_buff *skb; -struct sock *ssk;size_t psize; -long timeo; - -pr_debug ("msk=%p" , msk); -if (msk->subflow) { -pr_debug ("fallback passthrough" ); -return sock_sendmsg (msk->subflow, msg); - } - - ssk =mptcp_subflow_get_ref (msk); -if (!ssk) -return -ENOTCONN ; - -if (!msg_data_left (msg)) { -pr_debug ("empty send" ); - ret =sock_sendmsg (ssk->sk_socket, msg); - goto put_out; - } - -pr_debug ("conn_list->subflow=%p" , ssk); - -if (msg->msg_flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL )) { - ret = -ENOTSUPP ; - goto put_out; - } - -lock_sock (sk); -lock_sock (ssk); - timeo =sock_sndtimeo (sk, msg->msg_flags &MSG_DONTWAIT );/* use the mptcp page cache so that we can easily move the data * from one substream to another, but do per subflow memory accounting */ pfrag =sk_page_frag (sk);while (!sk_page_frag_refill (ssk, pfrag)) { - ret =sk_stream_wait_memory (ssk, &timeo); + ret =sk_stream_wait_memory (ssk, timeo);if (ret) - goto release_out; +return ret; } -/* Copy to page */ - poffset = pfrag->offset; +/* compute copy limit */ + mss_now =tcp_send_mss (ssk, &size_goal, msg->msg_flags); + psize =min_t (int, pfrag->size - pfrag->offset, size_goal); +pr_debug ("left=%zu" ,msg_data_left (msg)); - psize =copy_page_from_iter (pfrag->page, poffset, -min_t (size_t,msg_data_left (msg), - pfrag->size - poffset), + psize =copy_page_from_iter (pfrag->page, pfrag->offset, +min_t (size_t,msg_data_left (msg), psize), &msg->msg_iter);pr_debug ("left=%zu" ,msg_data_left (msg)); -if (!psize) { - ret = -EINVAL ; - goto release_out; - } +if (!psize) +return -EINVAL ;/* Mark the end of the previous write so the beginning of the * next write (with its own mptcp skb extension data) is not
@@ -117,12 +88,12 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)if (skb)TCP_SKB_CB (skb)->eor =1 ; - mss_now =tcp_send_mss (ssk, &size_goal, msg->msg_flags); - psize =min_t (int, size_goal, psize); - ret =do_tcp_sendpages (ssk, pfrag->page, poffset, psize, + ret =do_tcp_sendpages (ssk, pfrag->page, pfrag->offset, psize, msg->msg_flags |MSG_SENDPAGE_NOTLAST );if (ret <=0 ) - goto release_out; +return ret; +if (unlikely (ret < psize)) +iov_iter_revert (&msg->msg_iter, psize - ret);if (skb ==tcp_write_queue_tail (ssk))pr_err ("no new skb %p/%p" , sk, ssk);
@@ -150,8 +121,53 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)subflow_ctx (ssk)->rel_write_seq += ret;tcp_push (ssk, msg->msg_flags, mss_now,tcp_sk (ssk)->nonagle, size_goal); +return ret; +} + +static int mptcp_sendmsg (struct sock *sk,struct msghdr *msg,size_t len) +{ +struct mptcp_sock *msk =mptcp_sk (sk); +size_t copied =0 ; +struct sock *ssk; +int ret =0 ; +long timeo; + +pr_debug ("msk=%p" , msk); +if (msk->subflow) { +pr_debug ("fallback passthrough" ); +return sock_sendmsg (msk->subflow, msg); + } + + ssk =mptcp_subflow_get_ref (msk); +if (!ssk) +return -ENOTCONN ; + +if (!msg_data_left (msg)) { +pr_debug ("empty send" ); + ret =sock_sendmsg (ssk->sk_socket, msg); + goto put_out; + } + +pr_debug ("conn_list->subflow=%p" , ssk); + +if (msg->msg_flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL )) { + ret = -ENOTSUPP ; + goto put_out; + } + +lock_sock (sk); +lock_sock (ssk); + timeo =sock_sndtimeo (sk, msg->msg_flags &MSG_DONTWAIT ); +while (msg_data_left (msg)) { + ret =mptcp_sendmsg_frag (sk, ssk, msg, &timeo); +if (ret <0 ) +break ; + + copied += ret; + } +if (copied >0 ) + ret = copied; -release_out:release_sock (ssk);release_sock (sk);
Signature · 1 line
2.20.1
Paolo Abeni
PATCH v3 3/3 mptcp: allow collapsing consecutive sendpages on the same substream
In reply to Paolo AbeniIf the current sendmsg() lands on the same subflow we used last, we can try to collapse the data.
v2 -> v3:
- avoid BUG_ON(!mpext) on pending skb, we can have pure acks there with no mptcp ext
- rebased
v1 -> v2:
- reordered the code according to Mat's suggestion
- rebased on top of previous patch's changes
RFC -> v1:
- claried collasping schema
- update data_len for collapsed skb in mptcp_sendmsg_frag()
- fix collapsing decision on mptcp-level OoO
--- net/mptcp/protocol.c | 79 +++++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 19 deletions(-)
net/mptcp/protocol.c +60 -19
@@ -48,12 +48,25 @@ static struct sock *mptcp_subflow_get_ref(const struct mptcp_sock *msk)return NULL ; } +static inline bool mptcp_skb_can_collapse_to (const struct mptcp_sock *msk, +const struct sk_buff *skb, +const struct mptcp_ext *mpext) +{ +if (!tcp_skb_can_collapse_to (skb)) +return false; + +/* can collapse only if MPTCP level sequence is in order */ +return mpext && mpext->data_seq + mpext->data_len == msk->write_seq; +} +static int mptcp_sendmsg_frag (struct sock *sk,struct sock *ssk, -struct msghdr *msg,long *timeo) +struct msghdr *msg,long *timeo,int *pmss_now, +int *ps_goal) { +int mss_now, avail_size, size_goal, ret;struct mptcp_sock *msk =mptcp_sk (sk); +bool collapsed, can_collapse = false;struct mptcp_ext *mpext =NULL ; -int mss_now, size_goal, ret;struct page_frag *pfrag;struct sk_buff *skb;size_t psize;
@@ -70,8 +83,31 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,/* compute copy limit */ mss_now =tcp_send_mss (ssk, &size_goal, msg->msg_flags); - psize =min_t (int, pfrag->size - pfrag->offset, size_goal); + *pmss_now = mss_now; + *ps_goal = size_goal; + avail_size = size_goal; + skb =tcp_write_queue_tail (ssk); +if (skb) { + mpext =skb_ext_find (skb,SKB_EXT_MPTCP ); + +/* Limit the write to the size available in the +* current skb, if any, so that we create at most a new skb. +* If we run out of space in the current skb (e.g. the window +* size shrunk from last sent) a new skb will be allocated even +* is collapsing was allowed: collapsing is effectively +* disabled. +*/ + can_collapse =mptcp_skb_can_collapse_to (msk, skb, mpext); +if (!can_collapse) +TCP_SKB_CB (skb)->eor =1 ; +else if (size_goal - skb->len >0 ) + avail_size = size_goal - skb->len; +else + can_collapse = false; + } + psize =min_t (size_t, pfrag->size - pfrag->offset, avail_size); +/* Copy to page */ pr_debug ("left=%zu" ,msg_data_left (msg)); psize = copy_page_from_iter(pfrag->page, pfrag->offset, min_t(size_t, msg_data_left(msg), psize),
@@ -80,14 +116,9 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,if (!psize)return -EINVAL ; -/* Mark the end of the previous write so the beginning of the -* next write (with its own mptcp skb extension data) is not -* collapsed. +/* tell the TCP stack to delay the push so that we can safely +* access the skb after the sendpages call */ - skb =tcp_write_queue_tail (ssk); -if (skb) -TCP_SKB_CB (skb)->eor =1 ; - ret =do_tcp_sendpages (ssk, pfrag->page, pfrag->offset, psize, msg->msg_flags |MSG_SENDPAGE_NOTLAST );if (ret <=0 )
@@ -95,13 +126,16 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,if (unlikely (ret < psize))iov_iter_revert (&msg->msg_iter, psize - ret); -if (skb ==tcp_write_queue_tail (ssk)) -pr_err ("no new skb %p/%p" , sk, ssk); + collapsed = skb ==tcp_write_queue_tail (ssk); +BUG_ON (collapsed && !can_collapse); +if (collapsed) { +/* when collapsing mpext always exists */ + mpext->data_len += ret; + goto out; + } skb =tcp_write_queue_tail (ssk); - mpext =skb_ext_add (skb,SKB_EXT_MPTCP ); -if (mpext) {memset (mpext,0 ,sizeof (*mpext)); mpext->data_seq = msk->write_seq;
@@ -114,22 +148,25 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,pr_debug ("data_seq=%llu subflow_seq=%u data_len=%u checksum=%u, dsn64=%d" , mpext->data_seq, mpext->subflow_seq, mpext->data_len, mpext->checksum, mpext->dsn64); - }/* TODO: else fallback */ + } +/* TODO: else fallback; allocation can fail, but we can't easily retire +* skbs from the write_queue, as we need to roll-back TCP status +*/ +out: pfrag->offset += ret; msk->write_seq += ret;subflow_ctx (ssk)->rel_write_seq += ret; -tcp_push (ssk, msg->msg_flags, mss_now,tcp_sk (ssk)->nonagle, size_goal);return ret; }static int mptcp_sendmsg (struct sock *sk,struct msghdr *msg,size_t len) { +int mss_now =0 , size_goal =0 , ret =0 ;struct mptcp_sock *msk =mptcp_sk (sk);size_t copied =0 ;struct sock *ssk; -int ret =0 ;long timeo;pr_debug ("msk=%p" , msk);
@@ -159,14 +196,18 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)lock_sock (ssk); timeo =sock_sndtimeo (sk, msg->msg_flags &MSG_DONTWAIT );while (msg_data_left (msg)) { - ret =mptcp_sendmsg_frag (sk, ssk, msg, &timeo); + ret =mptcp_sendmsg_frag (sk, ssk, msg, &timeo, &mss_now, + &size_goal);if (ret <0 )break ; copied += ret; } -if (copied >0 ) +if (copied) { ret = copied; +tcp_push (ssk, msg->msg_flags, mss_now,tcp_sk (ssk)->nonagle, + size_goal); + }release_sock (ssk);release_sock (sk);
Signature · 1 line
2.20.1
Mat Martineau
Quoted 10 lines · Paolo Abeni
this series refactor the mptcp xmit path trying to make sendmsg behavior more consistent, memory efficent and improve its performances.
Self-test results are not affected: tests fail in similar way before and after this series.
To support
MP_JOINand multiple subflows, the idea is adding, with later patches, an RB-tree structure containing references to the pending page fragments, their length and the associated mptcpwrite_seqnumber (is subflowrel_write_seqneeded, too?).
rel_write_seq doesn't need to be stored for later, it will be different for every copy of the data that is sent.
Quoted 3 lines
RB-tree node can be allocated using the same
page frag used in this series for skb data. Each RB-tree node will own a reference to the page frag.
I think this is a nice improvement for the sendmsg code, the earlier issues are resolved and we should merge the series to mptcp_net-next.
Mat
Quoted 13 lines
v2 -> v3:
- rebased on top of "mptcp: refactor shutdown/close andconn_listhandling"
- cope with egres pkt with NULL extv1 -> v2:
- rebased on top of current git
- fix handling of some corner casesRFC -> v1:
- addressed Mat's feedback, dropping patch 3/4 and switching to per msk page frag use, and checking mptcp seq number for coalescing
Quoted 16 lines
Paolo Abeni (3):
mptcp: usesk_page_frag()in sendmsg
mptcp:sendmsg()do spool all the provided data
mptcp: allow collapsing consecutive sendpages on the same substream
net/mptcp/protocol.c| 190 ++++++++++++++++++++++++++++---------------1 file changed, 123 insertions(+), 67 deletions(-)Signature · 6 lines
2.20.1 _______________________________________________ mptcp mailing list mptcp(a)lists.01.org https://lists.01.org/mailman/listinfo/mptcp
--
Mat Martineau
Intel
Matthieu Baerts
Hi Paolo, Mat,
Quoted 18 lines · Mat Martineau
Quoted 11 lines · Paolo Abeni
this series refactor the mptcp xmit path trying to make sendmsg behavior more consistent, memory efficent and improve its performances.
Self-test results are not affected: tests fail in similar way before and after this series.
To support
MP_JOINand multiple subflows, the idea is adding, with later patches, an RB-tree structure containing references to the pending page fragments, their length and the associated mptcpwrite_seqnumber (is subflowrel_write_seqneeded, too?).
rel_write_seqdoesn't need to be stored for later, it will be different for every copy of the data that is sent.
Quoted 6 lines
Quoted 3 lines
RB-tree node can be allocated using the same
page frag used in this series for skb data. Each RB-tree node will own a reference to the page frag.I think this is a nice improvement for the sendmsg code, the earlier issues are resolved and we should merge the series to mptcp_net-next.
Thank you for the new version of the patches and the reviews. Just added at the end of the series.
https://github.com/multipath-tcp/mptcp_net-next/commits/export
Cheers,
Matt
Signature · 5 lines
Matthieu Baerts | R&D Engineer matthieu.baerts(a)tessares.net Tessares SA | Hybrid Access Solutions www.tessares.net 1 Avenue Jean Monnet, 1348 Louvain-la-Neuve, Belgium