net/tls: fixes for NVMe-over-TLS
Subsystems (5)
v5 cover net/tls: fixes for NVMe-over-TLS
Hi all,
here are some small fixes to get NVMe-over-TLS up and running. The first thre are just minor modifications to have MSG_EOR handled for TLS (and adding a test for it), but the last implements the ->read_sock() callback for tls_sw and I guess could do with some reviews.
It does work with my NVMe-TLS test harness, but what do I know :-)
As usual, comments and reviews are welcome.
Changes to the original submission:
- Add a testcase for MSG_EOR handling
Changes to v2:
- Bail out on conflicting message flags
- Rework flag handling
Changes to v3:
- Return -EINVAL on conflicting flags
- Rebase on top of net-next
Changes to v4:
- Add tlx_rx_reader_lock() to read_sock
- Add MSG_EOR handling to tls_sw_readpages()
Hannes Reinecke (4):
net/tls: handle MSG_EOR for tls_sw TX flow
net/tls: handle MSG_EOR for tls_device TX flow
selftests/net/tls: add test for MSG_EOR
net/tls: implement ->read_sock()
net/tls/tls.h | 2 + net/tls/tls_device.c | 25 +++++++-- net/tls/tls_main.c | 2 + net/tls/tls_sw.c | 87 +++++++++++++++++++++++++++++-- tools/testing/selftests/net/tls.c | 11 ++++ 5 files changed, 119 insertions(+), 8 deletions(-)
Signature · 1 line
2.35.3
PATCH v5 3/4 selftests/net/tls: add test for MSG_EOR
As the recent patch is modifying the behaviour for TLS re MSG_EOR handling we should be having a test for it.
--- tools/testing/selftests/net/tls.c | 11 +++++++++++ 1 file changed, 11 insertions(+)
tools/testing/selftests/net/tls.c +11 -0
@@ -477,6 +477,17 @@ TEST_F(tls, msg_more_unsent)EXPECT_EQ (recv (self->cfd, buf, send_len,MSG_DONTWAIT ),-1 ); } +TEST_F(tls, msg_eor) +{ +char const *test_str ="test_read" ; +int send_len =10 ; +char buf[10 ]; + +EXPECT_EQ (send (self->fd, test_str, send_len,MSG_EOR ), send_len); +EXPECT_EQ (recv (self->cfd, buf, send_len,MSG_WAITALL ), send_len); +EXPECT_EQ (memcmp (buf, test_str, send_len),0 ); +} + TEST_F(tls, sendmsg_single) {struct msghdr msg;
Signature · 1 line
2.35.3
PATCH v5 1/4 net/tls: handle MSG_EOR for tls_sw TX flow
In reply to Hannes Reinecketls_sw_sendmsg() / tls_do_sw_sendpage() already handles MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR. But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the negation of MSG_MORE. And erroring out if MSG_EOR is specified with either MSG_MORE or MSG_SENDPAGE_NOTLAST.
--- net/tls/tls_sw.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
net/tls/tls_sw.c +6 -3
@@ -984,6 +984,9 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,int ret =0 ;int pending; +if (!eor && (msg->msg_flags &MSG_EOR )) +return -EINVAL ; +if (unlikely (msg->msg_controllen)) { ret =tls_process_cmsg (sk, msg, &record_type);if (ret) {
@@ -1193,7 +1196,7 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)int ret;if (msg->msg_flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | -MSG_CMSG_COMPAT |MSG_SPLICE_PAGES | +MSG_CMSG_COMPAT |MSG_SPLICE_PAGES |MSG_EOR |MSG_SENDPAGE_NOTLAST |MSG_SENDPAGE_NOPOLICY ))return -EOPNOTSUPP ;
@@ -1287,7 +1290,7 @@ int tls_sw_sendpage_locked(struct sock *sk, struct page *page,struct bio_vec bvec;struct msghdr msg = { .msg_flags = flags |MSG_SPLICE_PAGES , }; -if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | +if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR |MSG_SENDPAGE_NOTLAST |MSG_SENDPAGE_NOPOLICY |MSG_NO_SHARED_FRAGS ))return -EOPNOTSUPP ;
@@ -1305,7 +1308,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,struct bio_vec bvec;struct msghdr msg = { .msg_flags = flags |MSG_SPLICE_PAGES , }; -if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL | +if (flags & ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_EOR |MSG_SENDPAGE_NOTLAST |MSG_SENDPAGE_NOPOLICY ))return -EOPNOTSUPP ;if (flags &MSG_SENDPAGE_NOTLAST )
Signature · 1 line
2.35.3
PATCH v5 2/4 net/tls: handle MSG_EOR for tls_device TX flow
In reply to Hannes Reinecketls_push_data() MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR.
But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the absence of MSG_MORE. Consequently we should return an error when both are set.
--- net/tls/tls_device.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-)
net/tls/tls_device.c +20 -5
@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,int copy, rc =0 ;long timeo; -if (flags & - ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST | -MSG_SPLICE_PAGES )) -return -EOPNOTSUPP ; -if (unlikely (sk->sk_err))return -sk->sk_err;
@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk, more = true;break ; } +if (flags &MSG_EOR ) { + more = false; +break ; + } done = true; }
@@ -582,6 +581,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)if (!tls_ctx->zerocopy_sendfile) msg->msg_flags &= ~MSG_SPLICE_PAGES ; +if (msg->msg_flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST | +MSG_SPLICE_PAGES |MSG_EOR )) +return -EOPNOTSUPP ; + +if ((msg->msg_flags & (MSG_MORE |MSG_EOR )) == (MSG_MORE |MSG_EOR )) +return -EOPNOTSUPP ; +mutex_lock (&tls_ctx->tx_lock);lock_sock (sk);
@@ -627,9 +634,17 @@ int tls_device_sendpage(struct sock *sk, struct page *page,struct bio_vec bvec;struct msghdr msg = { .msg_flags = flags |MSG_SPLICE_PAGES , }; +if (flags & + ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST | +MSG_SPLICE_PAGES |MSG_EOR )) +return -EOPNOTSUPP ; +if (flags &MSG_SENDPAGE_NOTLAST ) msg.msg_flags |=MSG_MORE ; +if ((msg.msg_flags & (MSG_MORE |MSG_EOR )) == (MSG_MORE |MSG_EOR )) +return -EINVAL ; +if (flags &MSG_OOB )return -EOPNOTSUPP ;
Signature · 1 line
2.35.3
PATCH v5 4/4 net/tls: implement ->read_sock()
In reply to Hannes ReineckeImplement ->read_sock() function for use with nvme-tcp.
--- net/tls/tls.h | 2 ++ net/tls/tls_main.c | 2 ++ net/tls/tls_sw.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+)
net/tls/tls.h +2 -0
@@ -114,6 +114,8 @@ bool tls_sw_sock_is_readable(struct sock *sk);ssize_t tls_sw_splice_read (struct socket *sock,loff_t *ppos,struct pipe_inode_info *pipe,size_t len,unsigned int flags); +int tls_sw_read_sock (struct sock *sk,read_descriptor_t *desc, +sk_read_actor_t read_actor);int tls_device_sendmsg (struct sock *sk,struct msghdr *msg,size_t size);void tls_device_splice_eof (struct socket *sock);
net/tls/tls_main.c +2 -0
@@ -963,10 +963,12 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG] ops[TLS_BASE ][TLS_SW ] = ops[TLS_BASE ][TLS_BASE ]; ops[TLS_BASE ][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_BASE ][TLS_SW ].poll = tls_sk_poll; + ops[TLS_BASE ][TLS_SW ].read_sock = tls_sw_read_sock; ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE ]; ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; + ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock;#ifdef CONFIG_TLS_DEVICE ops[TLS_HW ][TLS_BASE ] = ops[TLS_BASE ][TLS_BASE ];
net/tls/tls_sw.c +78 -0
84 lines not on this page · raw message
Signature · 1 line
2.35.3
Sagi Grimberg
maintainerRe: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 56 lines
Implement ->read_sock() function for use with nvme-tcp.
Signed-off-by Hannes Reinecke <[email protected]>Reviewed-by Sagi Grimberg <[email protected]>Cc Boris Pismenny <[email protected]>Cc Jakub Kicinski <[email protected]>--- net/tls/tls.h | 2 ++ net/tls/tls_main.c | 2 ++ net/tls/tls_sw.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+)net/tls/tls.h +2
-0@@ -114,6 +114,8 @@ bool tls_sw_sock_is_readable(struct sock *sk);ssize_t tls_sw_splice_read (struct socket *sock,loff_t *ppos,struct pipe_inode_info *pipe,size_t len,unsigned int flags); +int tls_sw_read_sock (struct sock *sk,read_descriptor_t *desc, +sk_read_actor_t read_actor);int tls_device_sendmsg (struct sock *sk,struct msghdr *msg,size_t size);void tls_device_splice_eof (struct socket *sock);net/tls/tls_main.c +2
-0@@ -963,10 +963,12 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG] ops[TLS_BASE ][TLS_SW ] = ops[TLS_BASE ][TLS_BASE ]; ops[TLS_BASE ][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_BASE ][TLS_SW ].poll = tls_sk_poll; + ops[TLS_BASE ][TLS_SW ].read_sock = tls_sw_read_sock; ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE ]; ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; + ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock;#ifdef CONFIG_TLS_DEVICE ops[TLS_HW ][TLS_BASE ] = ops[TLS_BASE ][TLS_BASE ];net/tls/tls_sw.c +4
-0@@ -2231,6 +2231,84 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, goto splice_read_end; } +int tls_sw_read_sock (struct sock *sk,read_descriptor_t *desc, +sk_read_actor_t read_actor) +{ +struct tls_context *tls_ctx =tls_get_ctx (sk);
net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247
Quoted 10 lines
net/tls/tls_sw.c +10
-0+struct tls_sw_context_rx *ctx =tls_sw_ctx_rx (tls_ctx); +struct strp_msg *rxm =NULL ; +struct tls_msg *tlm; +struct sk_buff *skb; +ssize_t copied =0 ; +int err, used; + + err =tls_rx_reader_lock (sk, ctx, true); +if (err <0 ) +return err;
Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?
I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.
Jakub Kicinski
maintainerRe: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 7 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247
Quoted 3 lines
net/tls/tls_sw.c +3
-0+ err =tls_rx_reader_lock (sk, ctx, true); +if (err <0 ) +return err;Unlike recvmsg or
splice_read, the caller ofread_sockis assumed to have the socket locked, andtls_rx_reader_lockalso callslock_sock, how is this not a deadlock?
Yeah :|
Quoted 2 lines
I'm not exactly clear why the lock is needed here or what is the subtle distinction between
tls_rx_reader_lockand whatlock_sockprovides.
It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.
IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.
The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.
Sabrina Dubroca
maintainerRe: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow
In reply to Hannes ReineckeQuoted 31 lines · Hannes Reinecke
tls_push_data()MSG_MORE/MSG_SENDPAGE_NOTLAST, but bails out onMSG_EOR.
But seeing thatMSG_EORis basically the opposite ofMSG_MORE/MSG_SENDPAGE_NOTLASTthis patch adds handlingMSG_EORby treating it as the absence ofMSG_MORE. Consequently we should return an error when both are set.Cc Jakub Kicinski <[email protected]>Signed-off-by Hannes Reinecke <[email protected]>--- net/tls/tls_device.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-)net/tls/tls_device.c +0
-5@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,int copy, rc =0 ;long timeo; -if (flags & - ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST | -MSG_SPLICE_PAGES )) -return -EOPNOTSUPP ; -if (unlikely (sk->sk_err))return -sk->sk_err;
net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 536
Quoted 6 lines
net/tls/tls_device.c +3
-0@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk, more = true;break ; } +if (flags &MSG_EOR ) { + more = false; +break ;
Why the break here? We don't want to close and push the record in that case? (the "if (done || ...)" block just below)
net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 540
Quoted 4 lines
net/tls/tls_device.c +1
-0+ } done = true; }
Thanks,
Signature · 1 line
Sabrina
Re: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow
Quoted 31 lines · Sabrina Dubroca
Quoted 30 lines · Hannes Reinecke
tls_push_data()MSG_MORE/MSG_SENDPAGE_NOTLAST, but bails out onMSG_EOR.
But seeing thatMSG_EORis basically the opposite ofMSG_MORE/MSG_SENDPAGE_NOTLASTthis patch adds handlingMSG_EORby treating it as the absence ofMSG_MORE. Consequently we should return an error when both are set.Cc Jakub Kicinski <[email protected]>Signed-off-by Hannes Reinecke <[email protected]>--- net/tls/tls_device.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-)net/tls/tls_device.c +0
-5@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,int copy, rc =0 ;long timeo; -if (flags & - ~(MSG_MORE |MSG_DONTWAIT |MSG_NOSIGNAL |MSG_SENDPAGE_NOTLAST | -MSG_SPLICE_PAGES )) -return -EOPNOTSUPP ; -if (unlikely (sk->sk_err))return -sk->sk_err;
Quoted 11 lines
net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 536
Quoted 7 lines
net/tls/tls_device.c +3
-0@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk, more = true;break ; } +if (flags &MSG_EOR ) { + more = false; +break ;Why the break here? We don't want to close and push the record in that case? (the "if (done || ...)" block just below)
Ah, yes, you are correct. Will be fixing it.
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Ivo Totev, Andrew Myers, Andrew McDonald, Martje Boudien Moerman
Re: [PATCH 4/4] net/tls: implement ->read_sock()
In reply to Jakub KicinskiQuoted 19 lines · Jakub Kicinski
Quoted 7 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247
Quoted 3 lines
net/tls/tls_sw.c +3
-0+ err =tls_rx_reader_lock (sk, ctx, true); +if (err <0 ) +return err;Unlike recvmsg or
splice_read, the caller ofread_sockis assumed to have the socket locked, andtls_rx_reader_lockalso callslock_sock, how is this not a deadlock?Yeah :|
Quoted 2 lines
I'm not exactly clear why the lock is needed here or what is the subtle distinction between
tls_rx_reader_lockand whatlock_sockprovides.It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.
Quoted 6 lines
IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.
The immediate need at the time was that if you did a
read()and someone else did apeek()at the same time from a stream of A B C D you may read A D B C.
Leaving me ever so confused.
read_sock() is a generic interface; we cannot require a protocol specific lock before calling it.
What to do now?
Drop the tls_rx_read_lock from read_sock() again?
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke Kernel Storage Architect [email protected] +49 911 74053 688 SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg HRB 36809 (AG Nürnberg), Geschäftsführer: Ivo Totev, Andrew Myers, Andrew McDonald, Martje Boudien Moerman
Sagi Grimberg
maintainerRe: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines
+ err =
tls_rx_reader_lock(sk, ctx, true);
+ if (err < 0)
+ return err;Unlike recvmsg or
splice_read, the caller ofread_sockis assumed to have the socket locked, andtls_rx_reader_lockalso callslock_sock, how is this not a deadlock?Yeah :|
Quoted 2 lines
I'm not exactly clear why the lock is needed here or what is the subtle distinction between
tls_rx_reader_lockand whatlock_sockprovides.It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.
Quoted 6 lines
IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.
The immediate need at the time was that if you did a
read()and someone else did apeek()at the same time from a stream of A B C D you may read A D B C.
Quoted 6 lines
Leaving me ever so confused.
read_sock()is a generic interface; we cannot require a protocol specific lock before calling it.What to do now?
Drop thetls_rx_read_lockfromread_sock()again?
Probably just need to synchronize the readers by splitting that from tls_rx_reader_lock:
--
net/tls/tls_sw.c +22 -17
@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot,return sk_flush_backlog (sk); } -static int tls_rx_reader_lock (struct sock *sk,struct tls_sw_context_rx *ctx, -bool nonblock) +static int tls_rx_reader_acquire (struct sock *sk,struct tls_sw_context_rx *ctx, +bool nonblock) {long timeo; -int err; - -lock_sock (sk); timeo =sock_rcvtimeo (sk, nonblock);
@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, !READ_ONCE(ctx->reader_present), &wait);remove_wait_queue (&ctx->wq, &wait); -if (timeo <=0 ) { - err = -EAGAIN ; - goto err_unlock; - } -if (signal_pending (current)) { - err =sock_intr_errno (timeo); - goto err_unlock; - } +if (timeo <=0 ) +return -EAGAIN ; +if (signal_pending (current)) +return sock_intr_errno (timeo); }WRITE_ONCE (ctx->reader_present,1 );return 0 ; +} -err_unlock: -release_sock (sk); +static int tls_rx_reader_lock (struct sock *sk,struct tls_sw_context_rx *ctx, +bool nonblock) +{ +int err; + +lock_sock (sk); + err =tls_rx_reader_acquire (sk, ctx, nonblock); +if (err) +release_sock (sk);return err; } -static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +static void tls_rx_reader_release (struct sock *sk,struct tls_sw_context_rx *ctx) {if (unlikely (ctx->reader_contended)) {if (wq_has_sleeper (&ctx->wq))
@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) }WRITE_ONCE (ctx->reader_present,0 ); +} + +static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +{ +tls_rx_reader_release (sk, ctx);release_sock (sk); } --
Then read_sock can just acquire/release.
Re: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 118 lines · Sagi Grimberg
Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines
+ err =
tls_rx_reader_lock(sk, ctx, true);
+ if (err < 0)
+ return err;Unlike recvmsg or
splice_read, the caller ofread_sockis assumed to have the socket locked, andtls_rx_reader_lockalso callslock_sock, how is this not a deadlock?Yeah :|
Quoted 2 lines
I'm not exactly clear why the lock is needed here or what is the subtle distinction between
tls_rx_reader_lockand whatlock_sockprovides.It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.
Quoted 6 lines
IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.
The immediate need at the time was that if you did a
read()and someone else did apeek()at the same time from a stream of A B C D you may read A D B C.Quoted 6 lines
Leaving me ever so confused.
read_sock()is a generic interface; we cannot require a protocol specific lock before calling it.What to do now?
Drop thetls_rx_read_lockfromread_sock()again?Probably just need to synchronize the readers by splitting that from
tls_rx_reader_lock:net/tls/tls_sw.c +2
-5@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, structtls_prot_info *prot, return sk_flush_backlog(sk); } -static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, - bool nonblock) +static int tls_rx_reader_acquire(struct sock *sk, structtls_sw_context_rx *ctx, + bool nonblock) { long timeo; - int err; - - lock_sock(sk);timeo =
sock_rcvtimeo(sk, nonblock);net/tls/tls_sw.c +15
-11@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk,struct tls_sw_context_rx *ctx, !READ_ONCE(ctx->reader_present), &wait); remove_wait_queue(&ctx->wq, &wait); - if (timeo <= 0) { - err = -EAGAIN; - goto err_unlock; - } - if (signal_pending(current)) { - err = sock_intr_errno(timeo); - goto err_unlock; - } + if (timeo <= 0) + return -EAGAIN; + if (signal_pending(current)) + return sock_intr_errno(timeo); } WRITE_ONCE(ctx->reader_present, 1); return 0; +} -err_unlock: - release_sock(sk); +static int tls_rx_reader_lock (struct sock *sk,struct tls_sw_context_rx *ctx, + bool nonblock) +{ + int err; + + lock_sock(sk); + err = tls_rx_reader_acquire(sk, ctx, nonblock); + if (err) + release_sock(sk); return err; } -static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +static void tls_rx_reader_release(struct sock *sk, struct
tls_sw_context_rx*ctx)
{
if (unlikely(ctx->reader_contended)) {
if (wq_has_sleeper(&ctx->wq))net/tls/tls_sw.c +5
-0@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) } WRITE_ONCE(ctx->reader_present, 0); +} + +static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +{ + tls_rx_reader_release(sk, ctx); release_sock(sk);
Quoted 4 lines
}
Signature · 1 line
Then read_sock can just acquire/release.
Good suggestion.
Will be including it in the next round.
Cheers,
Hannes
Sagi Grimberg
maintainerRe: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 122 lines · Hannes Reinecke
Quoted 121 lines · Sagi Grimberg
Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines
+ err =
tls_rx_reader_lock(sk, ctx, true);
+ if (err < 0)
+ return err;Unlike recvmsg or
splice_read, the caller ofread_sockis assumed to have the socket locked, andtls_rx_reader_lockalso callslock_sock, how is this not a deadlock?Yeah :|
Quoted 2 lines
I'm not exactly clear why the lock is needed here or what is the subtle distinction between
tls_rx_reader_lockand whatlock_sockprovides.It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.
Quoted 6 lines
IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.
The immediate need at the time was that if you did a
read()and someone else did apeek()at the same time from a stream of A B C D you may read A D B C.Quoted 6 lines
Leaving me ever so confused.
read_sock()is a generic interface; we cannot require a protocol specific lock before calling it.What to do now?
Drop thetls_rx_read_lockfromread_sock()again?Probably just need to synchronize the readers by splitting that from
tls_rx_reader_lock:net/tls/tls_sw.c +2
-5@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, structtls_prot_info *prot, return sk_flush_backlog(sk); } -static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx, - bool nonblock) +static int tls_rx_reader_acquire(struct sock *sk, struct tls_sw_context_rx *ctx, + bool nonblock) { long timeo; - int err; - - lock_sock(sk);timeo =
sock_rcvtimeo(sk, nonblock);net/tls/tls_sw.c +15
-11@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk,struct tls_sw_context_rx *ctx, !READ_ONCE(ctx->reader_present), &wait); remove_wait_queue(&ctx->wq, &wait); - if (timeo <= 0) { - err = -EAGAIN; - goto err_unlock; - } - if (signal_pending(current)) { - err = sock_intr_errno(timeo); - goto err_unlock; - } + if (timeo <= 0) + return -EAGAIN; + if (signal_pending(current)) + return sock_intr_errno(timeo); } WRITE_ONCE(ctx->reader_present, 1); return 0; +} -err_unlock: - release_sock(sk); +static int tls_rx_reader_lock (struct sock *sk,struct tls_sw_context_rx *ctx, + bool nonblock) +{ + int err; + + lock_sock(sk); + err = tls_rx_reader_acquire(sk, ctx, nonblock); + if (err) + release_sock(sk); return err; } -static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +static void tls_rx_reader_release(struct sock *sk, struct
tls_sw_context_rx*ctx)
{
if (unlikely(ctx->reader_contended)) {
if (wq_has_sleeper(&ctx->wq))net/tls/tls_sw.c +5
-0@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx) } WRITE_ONCE(ctx->reader_present, 0); +} + +static void tls_rx_reader_unlock (struct sock *sk,struct tls_sw_context_rx *ctx) +{ + tls_rx_reader_release(sk, ctx); release_sock(sk);}
Quoted 4 lines
Quoted 1 line
Then
read_sockcan just acquire/release.Good suggestion.
Will be including it in the next round.
Maybe more appropriate helper names would betls_rx_reader_enter / tls_rx_reader_exit.
Whatever Jakub prefers...
Jakub Kicinski
maintainerRe: [PATCH 4/4] net/tls: implement ->read_sock()
Quoted 7 lines · On Wed, 21 Jun 2023 12:49:21 +0300 Sagi Grimberg wrote:
Quoted 2 lines
Good suggestion.
Will be including it in the next round.Maybe more appropriate helper names would be
tls_rx_reader_enter/tls_rx_reader_exit.Whatever Jakub prefers...
I was thinking along the same lines but with __ in front of the names of the factored out code. Your naming as suggested in the diff is better.