Skip to contentSkip to conversation

net/tls: fixes for NVMe-over-TLS

PATCH Superseded Hannes Reinecke 4 participants 14 messages Jun 20, 2023 – Jun 21, 2023

Subsystems (5)

v5 of 9 Superseded Latest is v9 →

lore
Share & tools
lore
#0 lore raw

v5 cover net/tls: fixes for NVMe-over-TLS

Hi all,

here are some small fixes to get NVMe-over-TLS up and running. The first thre are just minor modifications to have MSG_EOR handled for TLS (and adding a test for it), but the last implements the ->read_sock() callback for tls_sw and I guess could do with some reviews.
It does work with my NVMe-TLS test harness, but what do I know :-)

As usual, comments and reviews are welcome.

Changes to the original submission:
- Add a testcase for MSG_EOR handling

Changes to v2:
- Bail out on conflicting message flags
- Rework flag handling

Changes to v3:
- Return -EINVAL on conflicting flags
- Rebase on top of net-next

Changes to v4:
- Add tlx_rx_reader_lock() to read_sock
- Add MSG_EOR handling to tls_sw_readpages()

Hannes Reinecke (4):
net/tls: handle MSG_EOR for tls_sw TX flow
net/tls: handle MSG_EOR for tls_device TX flow
selftests/net/tls: add test for MSG_EOR
net/tls: implement ->read_sock()

 net/tls/tls.h                     |  2 +
 net/tls/tls_device.c              | 25 +++++++--
 net/tls/tls_main.c                |  2 +
 net/tls/tls_sw.c                  | 87 +++++++++++++++++++++++++++++--
 tools/testing/selftests/net/tls.c | 11 ++++
 5 files changed, 119 insertions(+), 8 deletions(-)
Signature · 1 line
2.35.3
#5 lore raw

PATCH v5 3/4 selftests/net/tls: add test for MSG_EOR

As the recent patch is modifying the behaviour for TLS re MSG_EOR handling we should be having a test for it.

---
 tools/testing/selftests/net/tls.c | 11 +++++++++++
 1 file changed, 11 insertions(+)
tools/testing/selftests/net/tls.c +11 -0
index eccea9845c65..e8540fa5b310 100644
@@ -477,6 +477,17 @@ TEST_F(tls, msg_more_unsent)
 	EXPECT_EQ(recv(self->cfd, buf, send_len, MSG_DONTWAIT), -1);
 }
 
+TEST_F(tls, msg_eor)
+{
+	char const *test_str = "test_read";
+	int send_len = 10;
+	char buf[10];
+
+	EXPECT_EQ(send(self->fd, test_str, send_len, MSG_EOR), send_len);
+	EXPECT_EQ(recv(self->cfd, buf, send_len, MSG_WAITALL), send_len);
+	EXPECT_EQ(memcmp(buf, test_str, send_len), 0);
+}
+
 TEST_F(tls, sendmsg_single)
 {
 	struct msghdr msg;
Signature · 1 line
2.35.3
#1 lore raw

PATCH v5 1/4 net/tls: handle MSG_EOR for tls_sw TX flow

In reply to Hannes Reinecke

tls_sw_sendmsg() / tls_do_sw_sendpage() already handles MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR. But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the negation of MSG_MORE. And erroring out if MSG_EOR is specified with either MSG_MORE or MSG_SENDPAGE_NOTLAST.

---
 net/tls/tls_sw.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
net/tls/tls_sw.c +6 -3
index 319f61590d2c..97379e34c997 100644
@@ -984,6 +984,9 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg,
 	int ret = 0;
 	int pending;
 
+	if (!eor && (msg->msg_flags & MSG_EOR))
+		return -EINVAL;
+
 	if (unlikely(msg->msg_controllen)) {
 		ret = tls_process_cmsg(sk, msg, &record_type);
 		if (ret) {
@@ -1193,7 +1196,7 @@ int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 	int ret;
 
 	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
-			       MSG_CMSG_COMPAT | MSG_SPLICE_PAGES |
+			       MSG_CMSG_COMPAT | MSG_SPLICE_PAGES | MSG_EOR |
 			       MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
 		return -EOPNOTSUPP;
 
@@ -1287,7 +1290,7 @@ int tls_sw_sendpage_locked(struct sock *sk, struct page *page,
 	struct bio_vec bvec;
 	struct msghdr msg = { .msg_flags = flags | MSG_SPLICE_PAGES, };
 
-	if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
+	if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_EOR |
 		      MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY |
 		      MSG_NO_SHARED_FRAGS))
 		return -EOPNOTSUPP;
@@ -1305,7 +1308,7 @@ int tls_sw_sendpage(struct sock *sk, struct page *page,
 	struct bio_vec bvec;
 	struct msghdr msg = { .msg_flags = flags | MSG_SPLICE_PAGES, };
 
-	if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
+	if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_EOR |
 		      MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
 		return -EOPNOTSUPP;
 	if (flags & MSG_SENDPAGE_NOTLAST)
Signature · 1 line
2.35.3
#2 lore raw

PATCH v5 2/4 net/tls: handle MSG_EOR for tls_device TX flow

In reply to Hannes Reinecke

tls_push_data() MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR.
But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the absence of MSG_MORE. Consequently we should return an error when both are set.

---
 net/tls/tls_device.c | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)
net/tls/tls_device.c +20 -5
index b82770f68807..ebefd148ecf5 100644
@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,
 	int copy, rc = 0;
 	long timeo;
 
-	if (flags &
-	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST |
-	      MSG_SPLICE_PAGES))
-		return -EOPNOTSUPP;
-
 	if (unlikely(sk->sk_err))
 		return -sk->sk_err;
 
@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk,
 				more = true;
 				break;
 			}
+			if (flags & MSG_EOR) {
+				more = false;
+				break;
+			}
 
 			done = true;
 		}
@@ -582,6 +581,14 @@ int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 	if (!tls_ctx->zerocopy_sendfile)
 		msg->msg_flags &= ~MSG_SPLICE_PAGES;
 
+	if (msg->msg_flags &
+	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST |
+	      MSG_SPLICE_PAGES | MSG_EOR))
+		return -EOPNOTSUPP;
+
+	if ((msg->msg_flags & (MSG_MORE | MSG_EOR)) == (MSG_MORE | MSG_EOR))
+		return -EOPNOTSUPP;
+
 	mutex_lock(&tls_ctx->tx_lock);
 	lock_sock(sk);
 
@@ -627,9 +634,17 @@ int tls_device_sendpage(struct sock *sk, struct page *page,
 	struct bio_vec bvec;
 	struct msghdr msg = { .msg_flags = flags | MSG_SPLICE_PAGES, };
 
+	if (flags &
+	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST |
+	      MSG_SPLICE_PAGES | MSG_EOR))
+		return -EOPNOTSUPP;
+
 	if (flags & MSG_SENDPAGE_NOTLAST)
 		msg.msg_flags |= MSG_MORE;
 
+	if ((msg.msg_flags & (MSG_MORE | MSG_EOR)) == (MSG_MORE | MSG_EOR))
+		return -EINVAL;
+
 	if (flags & MSG_OOB)
 		return -EOPNOTSUPP;
 
Signature · 1 line
2.35.3
#6 lore raw

PATCH v5 4/4 net/tls: implement ->read_sock()

In reply to Hannes Reinecke

Implement ->read_sock() function for use with nvme-tcp.

---
 net/tls/tls.h      |  2 ++
 net/tls/tls_main.c |  2 ++
 net/tls/tls_sw.c   | 78 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 82 insertions(+)
net/tls/tls.h +2 -0
index d002c3af1966..ba55cd5c4913 100644
@@ -114,6 +114,8 @@ bool tls_sw_sock_is_readable(struct sock *sk);
 ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
 			   struct pipe_inode_info *pipe,
 			   size_t len, unsigned int flags);
+int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
+		     sk_read_actor_t read_actor);
 
 int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
 void tls_device_splice_eof(struct socket *sock);
net/tls/tls_main.c +2 -0
index 7b9c83dd7de2..1a062a8c6d33 100644
@@ -963,10 +963,12 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG]
 	ops[TLS_BASE][TLS_SW  ] = ops[TLS_BASE][TLS_BASE];
 	ops[TLS_BASE][TLS_SW  ].splice_read	= tls_sw_splice_read;
 	ops[TLS_BASE][TLS_SW  ].poll		= tls_sk_poll;
+	ops[TLS_BASE][TLS_SW  ].read_sock	= tls_sw_read_sock;
 
 	ops[TLS_SW  ][TLS_SW  ] = ops[TLS_SW  ][TLS_BASE];
 	ops[TLS_SW  ][TLS_SW  ].splice_read	= tls_sw_splice_read;
 	ops[TLS_SW  ][TLS_SW  ].poll		= tls_sk_poll;
+	ops[TLS_SW  ][TLS_SW  ].read_sock	= tls_sw_read_sock;
 
 #ifdef CONFIG_TLS_DEVICE
 	ops[TLS_HW  ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
net/tls/tls_sw.c +78 -0

84 lines not on this page · raw message

Signature · 1 line
2.35.3
maintainer
#7 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 56 lines

Implement ->read_sock() function for use with nvme-tcp.

---
  net/tls/tls.h      |  2 ++
  net/tls/tls_main.c |  2 ++
  net/tls/tls_sw.c   | 78 ++++++++++++++++++++++++++++++++++++++++++++++
  3 files changed, 82 insertions(+)
net/tls/tls.h +2 -0
index d002c3af1966..ba55cd5c4913 100644
@@ -114,6 +114,8 @@ bool tls_sw_sock_is_readable(struct sock *sk);
  ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
  			   struct pipe_inode_info *pipe,
  			   size_t len, unsigned int flags);
+int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
+		     sk_read_actor_t read_actor);
 
  int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
  void tls_device_splice_eof(struct socket *sock);
net/tls/tls_main.c +2 -0
index 7b9c83dd7de2..1a062a8c6d33 100644
@@ -963,10 +963,12 @@ static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG]
  	ops[TLS_BASE][TLS_SW  ] = ops[TLS_BASE][TLS_BASE];
  	ops[TLS_BASE][TLS_SW  ].splice_read	= tls_sw_splice_read;
  	ops[TLS_BASE][TLS_SW  ].poll		= tls_sk_poll;
+	ops[TLS_BASE][TLS_SW  ].read_sock	= tls_sw_read_sock;
 
  	ops[TLS_SW  ][TLS_SW  ] = ops[TLS_SW  ][TLS_BASE];
  	ops[TLS_SW  ][TLS_SW  ].splice_read	= tls_sw_splice_read;
  	ops[TLS_SW  ][TLS_SW  ].poll		= tls_sk_poll;
+	ops[TLS_SW  ][TLS_SW  ].read_sock	= tls_sw_read_sock;
 
  #ifdef CONFIG_TLS_DEVICE
  	ops[TLS_HW  ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
net/tls/tls_sw.c +4 -0
index 97379e34c997..e918c98bbeb2 100644
@@ -2231,6 +2231,84 @@ ssize_t tls_sw_splice_read(struct socket *sock,  loff_t *ppos,
  	goto splice_read_end;
  }
 
+int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
+		     sk_read_actor_t read_actor)
+{
+	struct tls_context *tls_ctx = tls_get_ctx(sk);

net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247

Quoted 10 lines
net/tls/tls_sw.c +10 -0
+	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
+	struct strp_msg *rxm = NULL;
+	struct tls_msg *tlm;
+	struct sk_buff *skb;
+	ssize_t copied = 0;
+	int err, used;
+
+	err = tls_rx_reader_lock(sk, ctx, true);
+	if (err < 0)
+		return err;

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

maintainer
#8 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 7 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:

net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247

Quoted 3 lines
net/tls/tls_sw.c +3 -0
+	err = tls_rx_reader_lock(sk, ctx, true);
+	if (err < 0)
+		return err;  

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

Yeah :|

Quoted 2 lines

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.

IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.

The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.

maintainer
#3 lore raw

Re: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow

In reply to Hannes Reinecke
Quoted 31 lines · Hannes Reinecke

tls_push_data() MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR.
But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the absence of MSG_MORE. Consequently we should return an error when both are set.

---
 net/tls/tls_device.c | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)
net/tls/tls_device.c +0 -5
index b82770f68807..ebefd148ecf5 100644
@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,
 	int copy, rc = 0;
 	long timeo;
 
-	if (flags &
-	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST |
-	      MSG_SPLICE_PAGES))
-		return -EOPNOTSUPP;
-
 	if (unlikely(sk->sk_err))
 		return -sk->sk_err;
 

net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 536

Quoted 6 lines
net/tls/tls_device.c +3 -0
@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk,
 				more = true;
 				break;
 			}
+			if (flags & MSG_EOR) {
+				more = false;
+				break;

Why the break here? We don't want to close and push the record in that case? (the "if (done || ...)" block just below)

net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 540

Quoted 4 lines
net/tls/tls_device.c +1 -0
+			}
 
 			done = true;
 		}

Thanks,

Signature · 1 line
Sabrina
#4 lore raw

Re: [PATCH 2/4] net/tls: handle MSG_EOR for tls_device TX flow

Quoted 31 lines · Sabrina Dubroca
Quoted 30 lines · Hannes Reinecke

tls_push_data() MSG_MORE / MSG_SENDPAGE_NOTLAST, but bails out on MSG_EOR.
But seeing that MSG_EOR is basically the opposite of MSG_MORE / MSG_SENDPAGE_NOTLAST this patch adds handling MSG_EOR by treating it as the absence of MSG_MORE. Consequently we should return an error when both are set.

---
  net/tls/tls_device.c | 25 ++++++++++++++++++++-----
  1 file changed, 20 insertions(+), 5 deletions(-)
net/tls/tls_device.c +0 -5
index b82770f68807..ebefd148ecf5 100644
@@ -440,11 +440,6 @@ static int tls_push_data(struct sock *sk,
  	int copy, rc = 0;
  	long timeo;
 
-	if (flags &
-	    ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST |
-	      MSG_SPLICE_PAGES))
-		return -EOPNOTSUPP;
-
  	if (unlikely(sk->sk_err))
  		return -sk->sk_err;
 
Quoted 11 lines

net/tls/tls_device.c static int tls_push_data(struct sock *sk, · line 536

Quoted 7 lines
net/tls/tls_device.c +3 -0
@@ -536,6 +531,10 @@ static int tls_push_data(struct sock *sk,
  				more = true;
  				break;
  			}
+			if (flags & MSG_EOR) {
+				more = false;
+				break;

Why the break here? We don't want to close and push the record in that case? (the "if (done || ...)" block just below)

Ah, yes, you are correct. Will be fixing it.

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke                Kernel Storage Architect
[email protected]                              +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), Geschäftsführer: Ivo Totev, Andrew
Myers, Andrew McDonald, Martje Boudien Moerman
#9 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

In reply to Jakub Kicinski
Quoted 19 lines · Jakub Kicinski
Quoted 7 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:

net/tls/tls_sw.c ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, · line 2247

Quoted 3 lines
net/tls/tls_sw.c +3 -0
+	err = tls_rx_reader_lock(sk, ctx, true);
+	if (err < 0)
+		return err;

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

Yeah :|

Quoted 2 lines

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.

Quoted 6 lines

IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.

The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.

Leaving me ever so confused.

read_sock() is a generic interface; we cannot require a protocol specific lock before calling it.

What to do now?
Drop the tls_rx_read_lock from read_sock() again?

Cheers,

Hannes

Signature · 5 lines
Dr. Hannes Reinecke                Kernel Storage Architect
[email protected]                              +49 911 74053 688
SUSE Software Solutions GmbH, Maxfeldstr. 5, 90409 Nürnberg
HRB 36809 (AG Nürnberg), Geschäftsführer: Ivo Totev, Andrew
Myers, Andrew McDonald, Martje Boudien Moerman
maintainer
#10 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines

+    err = tls_rx_reader_lock(sk, ctx, true);
+    if (err < 0)
+        return err;

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

Yeah :|

Quoted 2 lines

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.

Quoted 6 lines

IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.

The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.

Quoted 6 lines

Leaving me ever so confused.

read_sock() is a generic interface; we cannot require a protocol specific lock before calling it.

What to do now?
Drop the tls_rx_read_lock from read_sock() again?

Probably just need to synchronize the readers by splitting that from tls_rx_reader_lock:
--

net/tls/tls_sw.c +22 -17
index 53f944e6d8ef..53404c3fdcc6 100644
@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, struct tls_prot_info *prot,
        return sk_flush_backlog(sk);
 }
 
-static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx,
-                             bool nonblock)
+static int tls_rx_reader_acquire(struct sock *sk, struct tls_sw_context_rx *ctx,
+                            bool nonblock)
 {
        long timeo;
-       int err;
-
-       lock_sock(sk);
 
        timeo = sock_rcvtimeo(sk, nonblock);
 
@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx,
                              !READ_ONCE(ctx->reader_present), &wait);
                remove_wait_queue(&ctx->wq, &wait);
 
-               if (timeo <= 0) {
-                       err = -EAGAIN;
-                       goto err_unlock;
-               }
-               if (signal_pending(current)) {
-                       err = sock_intr_errno(timeo);
-                       goto err_unlock;
-               }
+               if (timeo <= 0)
+                       return -EAGAIN;
+               if (signal_pending(current))
+                       return sock_intr_errno(timeo);
        }
 
        WRITE_ONCE(ctx->reader_present, 1);
 
        return 0;
+}
 
-err_unlock:
-       release_sock(sk);
+static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx *ctx,
+                             bool nonblock)
+{
+       int err;
+
+       lock_sock(sk);
+       err = tls_rx_reader_acquire(sk, ctx, nonblock);
+       if (err)
+               release_sock(sk);
        return err;
 }
 
-static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx)
+static void tls_rx_reader_release(struct sock *sk, struct tls_sw_context_rx *ctx)
 {
        if (unlikely(ctx->reader_contended)) {
                if (wq_has_sleeper(&ctx->wq))
@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx)
        }
 
        WRITE_ONCE(ctx->reader_present, 0);
+}
+
+static void tls_rx_reader_unlock(struct sock *sk, struct tls_sw_context_rx *ctx)
+{
+       tls_rx_reader_release(sk, ctx);
        release_sock(sk);
 }
--

Then read_sock can just acquire/release.

#11 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 118 lines · Sagi Grimberg
Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines

+    err = tls_rx_reader_lock(sk, ctx, true);
+    if (err < 0)
+        return err;

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

Yeah :|

Quoted 2 lines

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.

Quoted 6 lines

IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.

The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.

Quoted 6 lines

Leaving me ever so confused.

read_sock() is a generic interface; we cannot require a protocol specific lock before calling it.

What to do now?
Drop the tls_rx_read_lock from read_sock() again?

Probably just need to synchronize the readers by splitting that from tls_rx_reader_lock:

net/tls/tls_sw.c +2 -5
index 53f944e6d8ef..53404c3fdcc6 100644
@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, struct 
 tls_prot_info *prot,
         return sk_flush_backlog(sk);
  }
 
-static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx 
 *ctx,
-                             bool nonblock)
+static int tls_rx_reader_acquire(struct sock *sk, struct 
 tls_sw_context_rx *ctx,
+                            bool nonblock)
  {
         long timeo;
-       int err;
-
-       lock_sock(sk);

        timeo = sock_rcvtimeo(sk, nonblock);

net/tls/tls_sw.c +15 -11
@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk, 
 struct tls_sw_context_rx *ctx,
                               !READ_ONCE(ctx->reader_present), &wait);
                 remove_wait_queue(&ctx->wq, &wait);
 
-               if (timeo <= 0) {
-                       err = -EAGAIN;
-                       goto err_unlock;
-               }
-               if (signal_pending(current)) {
-                       err = sock_intr_errno(timeo);
-                       goto err_unlock;
-               }
+               if (timeo <= 0)
+                       return -EAGAIN;
+               if (signal_pending(current))
+                       return sock_intr_errno(timeo);
         }
 
         WRITE_ONCE(ctx->reader_present, 1);
 
         return 0;
+}
 
-err_unlock:
-       release_sock(sk);
+static int tls_rx_reader_lock(struct sock *sk, struct tls_sw_context_rx 
 *ctx,
+                             bool nonblock)
+{
+       int err;
+
+       lock_sock(sk);
+       err = tls_rx_reader_acquire(sk, ctx, nonblock);
+       if (err)
+               release_sock(sk);
         return err;
  }
 
-static void tls_rx_reader_unlock(struct sock *sk, struct 
 tls_sw_context_rx *ctx)
+static void tls_rx_reader_release(struct sock *sk, struct 

tls_sw_context_rx *ctx)
 {
        if (unlikely(ctx->reader_contended)) {
                if (wq_has_sleeper(&ctx->wq))

net/tls/tls_sw.c +5 -0
@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock *sk, 
 struct tls_sw_context_rx *ctx)
         }
 
         WRITE_ONCE(ctx->reader_present, 0);
+}
+
+static void tls_rx_reader_unlock(struct sock *sk, struct 
 tls_sw_context_rx *ctx)
+{
+       tls_rx_reader_release(sk, ctx);
         release_sock(sk);
Quoted 4 lines

 }

Signature · 1 line
Then read_sock can just acquire/release.

Good suggestion.
Will be including it in the next round.

Cheers,

Hannes

maintainer
#12 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 122 lines · Hannes Reinecke
Quoted 121 lines · Sagi Grimberg
Quoted 27 lines
Quoted 20 lines · On Tue, 20 Jun 2023 16:21:22 +0300 Sagi Grimberg wrote:
Quoted 7 lines
Quoted 3 lines

+    err = tls_rx_reader_lock(sk, ctx, true);
+    if (err < 0)
+        return err;

Unlike recvmsg or splice_read, the caller of read_sock is assumed to have the socket locked, and tls_rx_reader_lock also calls lock_sock, how is this not a deadlock?

Yeah :|

Quoted 2 lines

I'm not exactly clear why the lock is needed here or what is the subtle distinction between tls_rx_reader_lock and what lock_sock provides.

It's a bit of a workaround for the consistency of the data stream. There's bunch of state in the TLS ULP and waiting for mem or data releases and re-takes the socket lock. So to stop the flow annoying corner case races I slapped a lock around all of the reader.

Quoted 6 lines

IMHO depending on the socket lock for anything non-trivial and outside of the socket itself is a bad idea in general.

The immediate need at the time was that if you did a read() and someone else did a peek() at the same time from a stream of A B C D you may read A D B C.

Quoted 6 lines

Leaving me ever so confused.

read_sock() is a generic interface; we cannot require a protocol specific lock before calling it.

What to do now?
Drop the tls_rx_read_lock from read_sock() again?

Probably just need to synchronize the readers by splitting that from tls_rx_reader_lock:

net/tls/tls_sw.c +2 -5
index 53f944e6d8ef..53404c3fdcc6 100644
@@ -1845,13 +1845,10 @@ tls_read_flush_backlog(struct sock *sk, struct 
 tls_prot_info *prot,
          return sk_flush_backlog(sk);
   }
 
-static int tls_rx_reader_lock(struct sock *sk, struct 
 tls_sw_context_rx *ctx,
-                             bool nonblock)
+static int tls_rx_reader_acquire(struct sock *sk, struct 
 tls_sw_context_rx *ctx,
+                            bool nonblock)
   {
          long timeo;
-       int err;
-
-       lock_sock(sk);

         timeo = sock_rcvtimeo(sk, nonblock);

net/tls/tls_sw.c +15 -11
@@ -1865,26 +1862,30 @@ static int tls_rx_reader_lock(struct sock *sk, 
 struct tls_sw_context_rx *ctx,
                                !READ_ONCE(ctx->reader_present), &wait);
                  remove_wait_queue(&ctx->wq, &wait);
 
-               if (timeo <= 0) {
-                       err = -EAGAIN;
-                       goto err_unlock;
-               }
-               if (signal_pending(current)) {
-                       err = sock_intr_errno(timeo);
-                       goto err_unlock;
-               }
+               if (timeo <= 0)
+                       return -EAGAIN;
+               if (signal_pending(current))
+                       return sock_intr_errno(timeo);
          }
 
          WRITE_ONCE(ctx->reader_present, 1);
 
          return 0;
+}
 
-err_unlock:
-       release_sock(sk);
+static int tls_rx_reader_lock(struct sock *sk, struct 
 tls_sw_context_rx *ctx,
+                             bool nonblock)
+{
+       int err;
+
+       lock_sock(sk);
+       err = tls_rx_reader_acquire(sk, ctx, nonblock);
+       if (err)
+               release_sock(sk);
          return err;
   }
 
-static void tls_rx_reader_unlock(struct sock *sk, struct 
 tls_sw_context_rx *ctx)
+static void tls_rx_reader_release(struct sock *sk, struct 

tls_sw_context_rx *ctx)
  {
         if (unlikely(ctx->reader_contended)) {
                 if (wq_has_sleeper(&ctx->wq))

net/tls/tls_sw.c +5 -0
@@ -1896,6 +1897,11 @@ static void tls_rx_reader_unlock(struct sock 
 *sk, struct tls_sw_context_rx *ctx)
          }
 
          WRITE_ONCE(ctx->reader_present, 0);
+}
+
+static void tls_rx_reader_unlock(struct sock *sk, struct 
 tls_sw_context_rx *ctx)
+{
+       tls_rx_reader_release(sk, ctx);
          release_sock(sk);

  }

Quoted 4 lines
Quoted 1 line

Then read_sock can just acquire/release.

Good suggestion.
Will be including it in the next round.

Maybe more appropriate helper names would be
tls_rx_reader_enter / tls_rx_reader_exit.

Whatever Jakub prefers...

maintainer
#13 lore raw

Re: [PATCH 4/4] net/tls: implement ->read_sock()

Quoted 7 lines · On Wed, 21 Jun 2023 12:49:21 +0300 Sagi Grimberg wrote:
Quoted 2 lines

Good suggestion.
Will be including it in the next round.

Maybe more appropriate helper names would be
tls_rx_reader_enter / tls_rx_reader_exit.

Whatever Jakub prefers...

I was thinking along the same lines but with __ in front of the names of the factored out code. Your naming as suggested in the diff is better.