net/tls: fixes for NVMe-over-TLS
Subsystems (5)
v3 cover net/tls: fixes for NVMe-over-TLS
Hi all,
here are some small fixes to get NVMe-over-TLS up and running. The first two are just minor modifications to have MSG_EOR handled for TLS, but the third implements the ->read_sock() callback for tls_sw and I guess could do with some reviews.
As usual, comments and reviews are welcome.
Changes to the original submission:
- Add a testcase for MSG_EOR handling
Changes to v2:
- Bail out on conflicting message flags
- Rework flag handling
Hannes Reinecke (4):
net/tls: handle MSG_EOR for tls_sw TX flow
net/tls: handle MSG_EOR for tls_device TX flow
net/tls: implement ->read_sock()
selftests/net/tls: add test for MSG_EOR
net/tls/tls.h | 2 + net/tls/tls_device.c | 24 +++++++-- net/tls/tls_main.c | 2 + net/tls/tls_sw.c | 88 +++++++++++++++++++++++++++++-- tools/testing/selftests/net/tls.c | 11 ++++ 5 files changed, 120 insertions(+), 7 deletions(-)
Signature · 1 line
2.35.3
PATCH v3 1/4 net/tls: handle MSG_EOR for tls_sw TX flow
tls_sw_sendmsg() / tls_do_sw_sendpage() already handles MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR. But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the negation of MSG_MORE. And erroring out if MSG_EOR is specified with either MSG_MORE or MSG_SENDPAGE_NOTLAST.
--- net/tls/tls_sw.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-)
net/tls/tls_sw.c +14 -3
@@ -953,7 +953,10 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)int pending;if (msg->msg_flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | -MSG_CMSG_COMPAT )) +MSG_EOR |MSG_CMSG_COMPAT )) +return -EOPNOTSUPP ; + +if (!eor && msg->msg_flags &MSG_EOR )return -EOPNOTSUPP ; ret =mutex_lock_interruptible (&tls_ctx->tx_lock);
@@ -1274,11 +1277,15 @@ static int tls_sw_do_sendpage(struct sock *sk, struct page *page,int tls_sw_sendpage_locked (struct sock *sk,struct page *page,int offset,size_t size,int flags) { -if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | +if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR |MSG_SENDPAGE_NOTLAST |MSG_SENDPAGE_NOPOLICY |MSG_NO_SHARED_FRAGS ))return -EOPNOTSUPP ; +if ((flags & (MSG_MORE |MSG_SENDPAGE_NOTLAST )) && + (flags &MSG_EOR )) +return -EINVAL ; +return tls_sw_do_sendpage (sk, page, offset, size, flags); }
@@ -1288,10 +1295,14 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,struct tls_context *tls_ctx =tls_get_ctx (sk);int ret; -if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | +if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR |MSG_SENDPAGE_NOTLAST |MSG_SENDPAGE_NOPOLICY ))return -EOPNOTSUPP ; +if ((flags & (MSG_MORE |MSG_SENDPAGE_NOTLAST )) && + (flags &MSG_EOR )) +return -EOPNOTSUPP ; + ret =mutex_lock_interruptible (&tls_ctx->tx_lock);if (ret)return ret;
Signature · 1 line
2.35.3
PATCH v3 2/4 net/tls: handle MSG_EOR for tls_device TX flow
In reply to Hannes Reinecketls_push_data() MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR.
But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the absence of MSG_MORE. Consequently we should return an error when both are set.
--- net/tls/tls_device.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-)
net/tls/tls_device.c +20 -4
@@ -448,10 +448,6 @@ static int tls_push_data(struct sock *sk,int copy, rc =0 ;long timeo; -if (flags & - ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST )) -return -EOPNOTSUPP ; -if (unlikely (sk->sk_err))return -sk->sk_err;
@@ -529,6 +525,10 @@ static int tls_push_data(struct sock *sk, more = true;break ; } +if (flags &MSG_EOR ) { + more = false; +break ; + } done = true; }
@@ -573,6 +573,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)union tls_iter_offset iter;int rc; +if (msg->msg_flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR )) +return -EOPNOTSUPP ; + +if ((msg->msg_flags &MSG_MORE ) && + (msg->msg_flags &MSG_EOR )) +return -EOPNOTSUPP ; +mutex_lock (&tls_ctx->tx_lock);lock_sock (sk);
@@ -601,9 +609,17 @@ int tls_device_sendpage(struct sock *sk, struct page *page,struct kvec iov;int rc; +if (flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST |MSG_EOR )) +return -EOPNOTSUPP ; +if (flags &MSG_SENDPAGE_NOTLAST ) flags |=MSG_MORE ; +if ((flags &MSG_MORE ) && + (flags &MSG_EOR )) +return -EOPNOTSUPP ; +mutex_lock (&tls_ctx->tx_lock);lock_sock (sk);
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow
Quoted 41 lines · Hannes Reinecke
tls_push_data()MSG_MORE/MSG_SENDPAGE_NOTLAST, but bails out onMSG_EOR.
But seeing thatMSG_EORis basically the opposite ofMSG_MORE/MSG_SENDPAGE_NOTLASTthis patch adds handlingMSG_EORby treating it as the absence ofMSG_MORE. Consequently we should return an error when both are set.Cc Jakub Kicinski <[email protected]>Signed-off-by Hannes Reinecke <[email protected]>--- net/tls/tls_device.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-)net/tls/tls_device.c +4
-4@@ -448,10 +448,6 @@ static int tls_push_data(struct sock *sk,int copy, rc =0 ;long timeo; -if (flags & - ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST )) -return -EOPNOTSUPP ; -if (unlikely (sk->sk_err))return -sk->sk_err;@@ -529,6 +525,10 @@ static int tls_push_data(struct sock *sk, more = true;break ; } +if (flags &MSG_EOR ) { + more = false; +break ; + } done = true; }
net/tls/tls_device.c int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) · line 582
Quoted 10 lines
net/tls/tls_device.c +7
-0@@ -573,6 +573,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)union tls_iter_offset iter;int rc; +if (msg->msg_flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR )) +return -EOPNOTSUPP ; + +if ((msg->msg_flags &MSG_MORE ) && + (msg->msg_flags &MSG_EOR )) +return -EOPNOTSUPP ;
EINVAL is more appropriate I think...
Quoted 8 lines
net/tls/tls_device.c +1
-0+mutex_lock (&tls_ctx->tx_lock);lock_sock (sk);@@ -601,9 +609,17 @@ int tls_device_sendpage(struct sock *sk, struct page *page,struct kvec iov;int rc;
net/tls/tls_device.c int tls_device_sendpage(struct sock *sk, struct page *page, · line 621
Quoted 10 lines
net/tls/tls_device.c +7
-0+if (flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST |MSG_EOR )) +return -EOPNOTSUPP ; +if (flags &MSG_SENDPAGE_NOTLAST ) flags |=MSG_MORE ; +if ((flags &MSG_MORE ) && + (flags &MSG_EOR )) +return -EOPNOTSUPP ;
EINVAL?
Re: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow
Quoted 46 lines · Sagi Grimberg
Quoted 43 lines · Hannes Reinecke
tls_push_data()MSG_MORE/MSG_SENDPAGE_NOTLAST, but bails out onMSG_EOR.
But seeing thatMSG_EORis basically the opposite ofMSG_MORE/MSG_SENDPAGE_NOTLASTthis patch adds handlingMSG_EORby treating it as the absence ofMSG_MORE. Consequently we should return an error when both are set.Cc Jakub Kicinski <[email protected]>Signed-off-by Hannes Reinecke <[email protected]>---
net/tls/tls_device.c| 24 ++++++++++++++++++++----1 file changed, 20 insertions(+), 4 deletions(-)net/tls/tls_device.c +5
-4@@ -448,10 +448,6 @@ static int tls_push_data(struct sock *sk, int copy, rc = 0; long timeo; - if (flags & - ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST)) - return -EOPNOTSUPP; - if (unlikely(sk->sk_err)) return -sk->sk_err;@@ -529,6 +525,10 @@ static int tls_push_data(struct sock *sk, more = true; break; } + if (flags & MSG_EOR) { + more = false; + break; + } done = true; }@@ -573,6 +573,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) union tls_iter_offset iter; int rc; + if (msg->msg_flags &
Quoted 9 lines
net/tls/tls_device.c int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) · line 582
Quoted 6 lines
net/tls/tls_device.c +6
-0+ ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_EOR)) + return -EOPNOTSUPP; + + if ((msg->msg_flags & MSG_MORE) && + (msg->msg_flags & MSG_EOR)) + return -EOPNOTSUPP;EINVAL is more appropriate I think...
Guess what, that's what I did initially.
But then when returning EINVAL we would arguably introduce a regression (as suddenly we'll be returning a different error code as previously). So with this patch we're backwards compatible.
But that's really a quesion for Jakub: what's more appropriate here? Return a new error code (which describes the situation better) or stick with the original one (and retain compability)?
Cheers,
Hannes
Jakub Kicinski
maintainerRe: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow
Quoted 6 lines · On Tue, 13 Jun 2023 10:11:01 +0200 Hannes Reinecke wrote:
Quoted 6 lines
Quoted 3 lines
+ if ((
msg->msg_flags&MSG_MORE) &&
+ (msg->msg_flags&MSG_EOR))
+ return-EOPNOTSUPP;EINVAL is more appropriate I think...
Quoted 8 lines
Guess what, that's what I did initially.
But then when returning EINVAL we would arguably introduce a regression (as suddenly we'll be returning a different error code as previously). So with this patch we're backwards compatible.But that's really a quesion for Jakub: what's more appropriate here? Return a new error code (which describes the situation better) or stick with the original one (and retain compability)?
EINVAL sounds better, EOPNOTSUPP means not implemented yet, once the thing is implemented it's natural that we'll start returning more precise error codes.
BTW you need to respin on top of net-next, David's multi-page sendpage has rejigged this code quite a bit.
PATCH v3 3/4 net/tls: implement ->read_sock()
In reply to Hannes ReineckeImplement ->read_sock() function for use with nvme-tcp.
--- net/tls/tls.h | 2 ++ net/tls/tls_main.c | 2 ++ net/tls/tls_sw.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+)
net/tls/tls.h +2 -0
@@ -113,6 +113,8 @@ bool tls_sw_sock_is_readable(struct sock *sk);ssize_t tls_sw_splice_read (struct socket *sock,loff_t *ppos,struct pipe_inode_info *pipe,size_t len,unsigned int flags); +int tls_sw_read_sock (struct sock *sk,read_descriptor_t *desc, +sk_read_actor_t read_actor);int tls_device_sendmsg (struct sock *sk,struct msghdr *msg,size_t size); int tls_device_sendpage(struct sock *sk, struct page *page,
net/tls/tls_main.c +2 -0
@@ -922,9 +922,11 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG] ops[TLS_BASE ][TLS_SW ] = ops[TLS_BASE ][TLS_BASE ]; ops[TLS_BASE ][TLS_SW ].splice_read = tls_sw_splice_read; + ops[TLS_BASE ][TLS_SW ].read_sock = tls_sw_read_sock; ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE ]; ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; + ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock;#ifdef CONFIG_TLS_DEVICE ops[TLS_HW ][TLS_BASE ] = ops[TLS_BASE ][TLS_BASE ];
net/tls/tls_sw.c +71 -0
77 lines not on this page · raw message
Signature · 1 line
2.35.3
PATCH v3 4/4 selftests/net/tls: add test for MSG_EOR
In reply to Hannes ReineckeAs the recent patch is modifying the behaviour for TLS re MSG_EOR handling we should be having a test for it.
--- tools/testing/selftests/net/tls.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
tools/testing/selftests/net/tls.c +11 -0
@@ -476,6 +476,17 @@ TEST_F(tls, msg_more_unsent)EXPECT_EQ (recv (self->cfd, buf, send_len,MSG_DONTWAIT ),-1 ); } +TEST_F(tls, msg_eor) +{ +char const *test_str ="test_read" ; +int send_len =10 ; +char buf[10 ]; + +EXPECT_EQ (send (self->fd, test_str, send_len,MSG_EOR ), send_len); +EXPECT_EQ (recv (self->cfd, buf, send_len,MSG_WAITALL ), send_len); +EXPECT_EQ (memcmp (buf, test_str, send_len),0 ); +} + TEST_F(tls, sendmsg_single) {struct msghdr msg;
Signature · 1 line
2.35.3