Skip to contentSkip to conversation

NVMe/TLS connection issues to SPDK

PATCH In review Chris Leech 4 participants 22 messages Jul 21, 2025 – Aug 26, 2025

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover NVMe/TLS connection issues to SPDK

I was attempting to debug connecting the Linux driver / libnvme / ktls-utils host stack to the SPDK nvmf_tgt over TLS, and ran into some issues.

The TLS connection fails to complete a handshake because the TLS PSKs are different. The NVMe/TCP specified key derivation steps from the configured interchange format, to a retained PSK and finally the TLS PSK, is implemented incompatibly in libnvme and SPDK. After some investigation, I believe the SPDK implementation to be correct and am providing a libnvme patch to match it. With libnvme modified, I see the TLS handshake complete in tlshd.

(Note that this was tested using the obsolete "version 0" PSK Identity and TLS PSK derivation from the TCP transport 1.0 specification, as SPDK has not been updated with the "version 1" changes)

The NVMe/TCP host driver then quickly fails when SPDK sends a TLS "New Session Ticket" message before ICResp.

While possibly pointless due to the transport specification prohibition on session resumption and 0-RTT data, I don't think this is necessarily wrong and the host driver should be able to safely ignore it and continue.

I'm working on testing that out, but a more general TLS message demuxing layer to deal with post-handshake messages other than application data may be wanted to avoid sprinkling checks around the nvme driver.

Chris Leech (1):
libnvme: TLS PSK derivation fixes

 src/nvme/linux.c | 86 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 29 deletions(-)
Signature · 1 line
2.50.1
#1 lore raw

PATCH 1/1 libnvme: TLS PSK derivation fixes

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.

1) The 16-bit HkdfLabel.length value must be converted to network byte order.

2) The variable length HkdfLabel.label and HkdfLabel.context vectors must be prefixed with a length byte.

---
 src/nvme/linux.c | 86 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 29 deletions(-)
src/nvme/linux.c +57 -29
index ae4aa526..674b20b5 100644
@@ -618,6 +618,46 @@ static DEFINE_CLEANUP_FUNC(
 	cleanup_evp_pkey_ctx, EVP_PKEY_CTX *, EVP_PKEY_CTX_free)
 #define _cleanup_evp_pkey_ctx_ __cleanup__(cleanup_evp_pkey_ctx)
 
+/*
+ * hkdf_info_printf()
+ *
+ * Helper function to append variable length label and context to an HkdfLabel
+ *
+ * RFC 8446 (TLS 1.3) Section 7.1 defines the HKDF-Expand-Label function as a
+ * specialization of the HKDF-Expand function (RFC 5869), where the info
+ * parameter is structured as an HkdfLabel.
+ *
+ * An HkdfLabel structure includes two variable length vectors (label and
+ * context) which must be preceded by their content length as per RFC 8446
+ * Section 3.4 (and not NUL terminated as per Section 7.1). Additionally,
+ * HkdfLabel.label must begin with "tls13 "
+ *
+ * Returns the number of bytes appended to the HKDF info buffer, or -1 on an
+ * error.
+ */
+__attribute__((format(printf, 2, 3)))
+static int hkdf_info_printf(EVP_PKEY_CTX *ctx, char *fmt, ...)
+{
+	_cleanup_free_ char *str;
+	uint8_t len;
+	int ret;
+
+	va_list myargs;
+	va_start(myargs, fmt);
+	ret = vasprintf(&str, fmt, myargs);
+	va_end(myargs);
+	if (ret < 0)
+		return ret;
+	if (ret > 255)
+		return -1;
+	len = ret;
+	if (EVP_PKEY_CTX_add1_hkdf_info(ctx, (unsigned char *)&len, 1) <= 0)
+		return -1;
+	if (EVP_PKEY_CTX_add1_hkdf_info(ctx, (unsigned char *)str, len) <= 0)
+		return -1;
+	return (ret + 1);
+}
+
 /*
  * derive_retained_key()
  *
@@ -652,7 +692,7 @@ static int derive_retained_key(int hmac, const char *hostnqn,
 			       size_t key_len)
 {
 	_cleanup_evp_pkey_ctx_ EVP_PKEY_CTX *ctx = NULL;
-	uint16_t length = key_len & 0xFFFF;
+	uint16_t length = htons(key_len & 0xFFFF);
 	const EVP_MD *md;
 	size_t hmac_len;
 
@@ -690,18 +730,11 @@ static int derive_retained_key(int hmac, const char *hostnqn,
 		errno = ENOKEY;
 		return -1;
 	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"tls13 ", 6) <= 0) {
-		errno = ENOKEY;
-		return -1;
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"HostNQN", 7) <= 0) {
+	if (hkdf_info_printf(ctx, "tls13 HostNQN") <= 0) {
 		errno = ENOKEY;
 		return -1;
 	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)hostnqn, strlen(hostnqn)) <= 0) {
+	if (hkdf_info_printf(ctx, "%s", hostnqn) <= 0) {
 		errno = ENOKEY;
 		return -1;
 	}
@@ -736,12 +769,13 @@ static int derive_retained_key(int hmac, const char *hostnqn,
  *
  * and the value '0' is invalid here.
  */
+
 static int derive_tls_key(int version, unsigned char cipher,
 			  const char *context, unsigned char *retained,
 			  unsigned char *psk, size_t key_len)
 {
 	_cleanup_evp_pkey_ctx_ EVP_PKEY_CTX *ctx = NULL;
-	uint16_t length = key_len & 0xFFFF;
+	uint16_t length = htons(key_len & 0xFFFF);
 	const EVP_MD *md;
 	size_t hmac_len;
 
@@ -774,30 +808,24 @@ static int derive_tls_key(int version, unsigned char cipher,
 		errno = ENOKEY;
 		return -1;
 	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"tls13 ", 6) <= 0) {
-		errno = ENOKEY;
-		return -1;
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"nvme-tls-psk", 12) <= 0) {
+	if (hkdf_info_printf(ctx, "tls13 nvme-tls-psk") <= 0) {
 		errno = ENOKEY;
 		return -1;
 	}
-	if (version == 1) {
-		char hash_str[5];
-
-		sprintf(hash_str, "%02d ", cipher);
-		if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-				(const unsigned char *)hash_str,
-				strlen(hash_str)) <= 0) {
+	switch (version) {
+	case 0:
+		if (hkdf_info_printf(ctx, "%s", context) <= 0) {
 			errno = ENOKEY;
 			return -1;
 		}
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)context,
-			strlen(context)) <= 0) {
+		break;
+	case 1:
+		if (hkdf_info_printf(ctx, "%02d %s", cipher, context) <= 0) {
+			errno = ENOKEY;
+			return -1;
+		}
+		break;
+	default:
 		errno = ENOKEY;
 		return -1;
 	}
Signature · 1 line
2.50.1
Reviewed-by on v1 1/1
#2 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 144 lines · Chris Leech

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.

1) The 16-bit HkdfLabel.length value must be converted to network byte order.

2) The variable length HkdfLabel.label and HkdfLabel.context vectors must be prefixed with a length byte.

---
  src/nvme/linux.c | 86 ++++++++++++++++++++++++++++++++----------------
  1 file changed, 57 insertions(+), 29 deletions(-)
src/nvme/linux.c +50 -29
index ae4aa526..674b20b5 100644
@@ -618,6 +618,46 @@ static DEFINE_CLEANUP_FUNC(
  	cleanup_evp_pkey_ctx, EVP_PKEY_CTX *, EVP_PKEY_CTX_free)
  #define _cleanup_evp_pkey_ctx_ __cleanup__(cleanup_evp_pkey_ctx)
 
+/*
+ * hkdf_info_printf()
+ *
+ * Helper function to append variable length label and context to an HkdfLabel
+ *
+ * RFC 8446 (TLS 1.3) Section 7.1 defines the HKDF-Expand-Label function as a
+ * specialization of the HKDF-Expand function (RFC 5869), where the info
+ * parameter is structured as an HkdfLabel.
+ *
+ * An HkdfLabel structure includes two variable length vectors (label and
+ * context) which must be preceded by their content length as per RFC 8446
+ * Section 3.4 (and not NUL terminated as per Section 7.1). Additionally,
+ * HkdfLabel.label must begin with "tls13 "
+ *
+ * Returns the number of bytes appended to the HKDF info buffer, or -1 on an
+ * error.
+ */
+__attribute__((format(printf, 2, 3)))
+static int hkdf_info_printf(EVP_PKEY_CTX *ctx, char *fmt, ...)
+{
+	_cleanup_free_ char *str;
+	uint8_t len;
+	int ret;
+
+	va_list myargs;
+	va_start(myargs, fmt);
+	ret = vasprintf(&str, fmt, myargs);
+	va_end(myargs);
+	if (ret < 0)
+		return ret;
+	if (ret > 255)
+		return -1;
+	len = ret;
+	if (EVP_PKEY_CTX_add1_hkdf_info(ctx, (unsigned char *)&len, 1) <= 0)
+		return -1;
+	if (EVP_PKEY_CTX_add1_hkdf_info(ctx, (unsigned char *)str, len) <= 0)
+		return -1;
+	return (ret + 1);
+}
+
  /*
   * derive_retained_key()
   *
@@ -652,7 +692,7 @@ static int derive_retained_key(int hmac, const char *hostnqn,
  			       size_t key_len)
  {
  	_cleanup_evp_pkey_ctx_ EVP_PKEY_CTX *ctx = NULL;
-	uint16_t length = key_len & 0xFFFF;
+	uint16_t length = htons(key_len & 0xFFFF);
  	const EVP_MD *md;
  	size_t hmac_len;
 
@@ -690,18 +730,11 @@ static int derive_retained_key(int hmac, const char *hostnqn,
  		errno = ENOKEY;
  		return -1;
  	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"tls13 ", 6) <= 0) {
-		errno = ENOKEY;
-		return -1;
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"HostNQN", 7) <= 0) {
+	if (hkdf_info_printf(ctx, "tls13 HostNQN") <= 0) {
  		errno = ENOKEY;
  		return -1;
  	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)hostnqn, strlen(hostnqn)) <= 0) {
+	if (hkdf_info_printf(ctx, "%s", hostnqn) <= 0) {
  		errno = ENOKEY;
  		return -1;
  	}
@@ -736,12 +769,13 @@ static int derive_retained_key(int hmac, const char *hostnqn,
   *
   * and the value '0' is invalid here.
   */
+
  static int derive_tls_key(int version, unsigned char cipher,
  			  const char *context, unsigned char *retained,
  			  unsigned char *psk, size_t key_len)
  {
  	_cleanup_evp_pkey_ctx_ EVP_PKEY_CTX *ctx = NULL;
-	uint16_t length = key_len & 0xFFFF;
+	uint16_t length = htons(key_len & 0xFFFF);
  	const EVP_MD *md;
  	size_t hmac_len;
 
@@ -774,30 +808,24 @@ static int derive_tls_key(int version, unsigned char cipher,
  		errno = ENOKEY;
  		return -1;
  	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"tls13 ", 6) <= 0) {
-		errno = ENOKEY;
-		return -1;
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)"nvme-tls-psk", 12) <= 0) {
+	if (hkdf_info_printf(ctx, "tls13 nvme-tls-psk") <= 0) {
  		errno = ENOKEY;
  		return -1;
  	}
-	if (version == 1) {
-		char hash_str[5];
-
-		sprintf(hash_str, "%02d ", cipher);
-		if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-				(const unsigned char *)hash_str,
-				strlen(hash_str)) <= 0) {
+	switch (version) {
+	case 0:
+		if (hkdf_info_printf(ctx, "%s", context) <= 0) {
  			errno = ENOKEY;
  			return -1;
  		}
-	}
-	if (EVP_PKEY_CTX_add1_hkdf_info(ctx,
-			(const unsigned char *)context,
-			strlen(context)) <= 0) {
+		break;

src/nvme/linux.c static int derive_tls_key(int version, unsigned char cipher, · line 831

Quoted 10 lines
src/nvme/linux.c +7 -0
+	case 1:
+		if (hkdf_info_printf(ctx, "%02d %s", cipher, context) <= 0) {
+			errno = ENOKEY;
+			return -1;
+		}
+		break;
+	default:
  		errno = ENOKEY;
  		return -1;
  	}

Hmm. I _thought_ we had it all fixed...

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#3 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 6 lines · Hannes Reinecke
Quoted 4 lines · Chris Leech

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.
...

Hmm. I _thought_ we had it all fixed...

I went into this expecting/hoping to find the problem on the spdk side, and I'd be happy to wrong here (especially if backed up by another interoperable implementor).

The lack of a full example in the nvme/tcp transport spec to verify implemenatations against is kind of a bummer.

- Chris

#4 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 6 lines · Chris Leech
Quoted 5 lines · Hannes Reinecke
Quoted 4 lines · Chris Leech

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.
...

Quoted 9 lines
Quoted 1 line

Hmm. I _thought_ we had it all fixed...

I went into this expecting/hoping to find the problem on the spdk side, and I'd be happy to wrong here (especially if backed up by another interoperable implementor).

The lack of a full example in the nvme/tcp transport spec to verify implemenatations against is kind of a bummer.

Okay, seems that you have been right after all.
I've re-read RFC 8466 and indeed you seem to be right about the encoding of variable length vectors (cf RFC 8446 section 3.4).

So I guess we need to take this patch after all.

_However_: PSKs generated with after applying this patch will be different than those prior to this patch.
Consequently there will be interop issues with existing implementations (which will use the original encoding).

I guess we would need to wait for the target implementations to be fixed
or introduce a flag switching to the old / compat implementation to avoid interop issues.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#5 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 13 lines · Hannes Reinecke
Quoted 12 lines · Chris Leech
Quoted 6 lines · Hannes Reinecke
Quoted 4 lines · Chris Leech

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.
...

Hmm. I _thought_ we had it all fixed...

I went into this expecting/hoping to find the problem on the spdk side, and I'd be happy to wrong here (especially if backed up by another interoperable implementor).

Quoted 8 lines
Quoted 3 lines

The lack of a full example in the nvme/tcp transport spec to verify implemenatations against is kind of a bummer.

Okay, seems that you have been right after all.
I've re-read RFC 8466 and indeed you seem to be right about the encoding of variable length vectors (cf RFC 8446 section 3.4).

So I guess we need to take this patch after all.

So we're back to agreeing on the correctness of the changes? Honestly, I appreciate the scrutiny.

Quoted 8 lines

_However_: PSKs generated with after applying this patch will be different than those prior to this patch.
Consequently there will be interop issues with existing implementations (which will use the original encoding).

I guess we would need to wait for the target implementations to be fixed or introduce a flag switching to the old / compat implementation to avoid interop issues.

Yes, it is a breaking change for compatibility with other out-of-spec implementations. I'm hesitant to carry a flag for that, but it wouldn't be too much trouble to implement.

Just don't touch your keyfile? The tls-key import/export commands are operating on the final derived TLS PSKs right?

- Chris

#6 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 17 lines · Chris Leech
Quoted 16 lines · Hannes Reinecke
Quoted 15 lines · Chris Leech
Quoted 6 lines · Hannes Reinecke
Quoted 4 lines · Chris Leech

There are issues with the Retained and TLS PSK derivations due to the implementation not adhering to the RFC 8446 definition of the HKDF-Expand-Label function.
...

Hmm. I _thought_ we had it all fixed...

I went into this expecting/hoping to find the problem on the spdk side, and I'd be happy to wrong here (especially if backed up by another interoperable implementor).

The lack of a full example in the nvme/tcp transport spec to verify implemenatations against is kind of a bummer.

Quoted 9 lines
Quoted 5 lines

Okay, seems that you have been right after all.
I've re-read RFC 8466 and indeed you seem to be right about the encoding of variable length vectors (cf RFC 8446 section 3.4).

So I guess we need to take this patch after all.

So we're back to agreeing on the correctness of the changes? Honestly, I appreciate the scrutiny.

Yeah, sorry. I indeed misread (or rather ignored) section 3 from the RFC when coding the key derivation.
You are correct, and we need to prefix each string with the length.

But: this is an incompatible change. Once we integrate this patch the keyring will end up with a _different_ derived PSK than it would without that patch. So the _same_ PSK (in interchange format) will result in different PSKs in the keyring, causing the handshake to fail if the other side is not aware of that.

Quoted 9 lines
Quoted 8 lines

_However_: PSKs generated with after applying this patch will be different than those prior to this patch.
Consequently there will be interop issues with existing implementations (which will use the original encoding).

I guess we would need to wait for the target implementations to be fixed or introduce a flag switching to the old / compat implementation to avoid interop issues.

Quoted 7 lines

Yes, it is a breaking change for compatibility with other out-of-spec implementations. I'm hesitant to carry a flag for that, but it wouldn't be too much trouble to implement.

Just don't touch your keyfile? The tls-key import/export commands are operating on the final derived TLS PSKs right?

That might be the way out, and in fact what we should be doing. However, the fact still remains: after this patch has been applied one _cannot_ generate new PSKs until the other side has been updated, too.
For linux it's easy, we can just require 'version xyz' from libnvme and the issue is solved. But for other implementations? There are IHVs out there who already shipped their products with TLS enablement, and they need to be fixed, too.
And their timeline is vastly longer than ours.

So to avoid us having to synchronize against all of the others I think it might be easier to add a 'compat' flag of sorts to generate PSKs with the 'original' derivation algorithm, and then increase the libnvme version number once it's in.
Then we can point the IHVs to that number so that they reference that version once their firmware is updated.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#7 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 6 lines · Hannes Reinecke

So to avoid us having to synchronize against all of the others I think it might be easier to add a 'compat' flag of sorts to generate PSKs with the 'original' derivation algorithm, and then increase the libnvme version number once it's in.
Then we can point the IHVs to that number so that they reference that version once their firmware is updated.

This is a really bad precedent. Do you really want to create a new QUIK nightmare for NVMe/TLS?

If you add a compat flag they you will NEVER be able to get rid of it.

We should fix this correctly now and break backwards compatibility. This will force implementations to do it right.

Do it now or it will never happen.

/John

Quoted 5 lines

Cheers,

Hannes

#8 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

In reply to Hannes Reinecke
Quoted 33 lines · Hannes Reinecke

...
But: this is an incompatible change. Once we integrate this patch the keyring will end up with a _different_ derived PSK than it would without that patch. So the _same_ PSK (in interchange format) will result in different PSKs in the keyring, causing the handshake to fail if the other side is not aware of that.

Quoted 9 lines
Quoted 8 lines

_However_: PSKs generated with after applying this patch will be different than those prior to this patch.
Consequently there will be interop issues with existing implementations (which will use the original encoding).

I guess we would need to wait for the target implementations to be fixed or introduce a flag switching to the old / compat implementation to avoid interop issues.

Quoted 7 lines

Yes, it is a breaking change for compatibility with other out-of-spec implementations. I'm hesitant to carry a flag for that, but it wouldn't be too much trouble to implement.

Just don't touch your keyfile? The tls-key import/export commands are operating on the final derived TLS PSKs right?

That might be the way out, and in fact what we should be doing. However, the fact still remains: after this patch has been applied one _cannot_ generate new PSKs until the other side has been updated, too.
For linux it's easy, we can just require 'version xyz' from libnvme and the issue is solved. But for other implementations? There are IHVs out there who already shipped their products with TLS enablement, and they need to be fixed, too.
And their timeline is vastly longer than ours.

Quoted 6 lines

So to avoid us having to synchronize against all of the others I think it might be easier to add a 'compat' flag of sorts to generate PSKs with the 'original' derivation algorithm, and then increase the libnvme version number once it's in.
Then we can point the IHVs to that number so that they reference that version once their firmware is updated.

I've spent a bit of time on this, and while I don't have patches ready to share just yet I do think it's worth commenting on the scope of the change to keep the existing functionality behind a cli switch.

libnvme:
- keeping both sets of key derivation functions (this is not bad)
- need to expose that functionality, so new exported APIs
(I was thinking something like a flag field over a legacy-only function)
- #define NVME_TLS_PSK_LEGACY (1u << 0)
- char *nvme_generate_tls_key_identity_2(..., unsigned int flags)
- long nvme_insert_tls_key_versioned_2(..., unsigned int flags)

- support paths that load keys through nvme_fabrics_cfg [1] (for nvme-cli fabrics commands that take --tls_key)
- add some sort of flag there, like nvme_fabrics_cfg.tls_legacy
- add to merge_config, nvmf_update_config, build_options, supported_options, etc.

- the JSON schema/code would need to support this for persistent configurations

nvme-cli:
- add a command line flag to keyring command functions
- convert to use of new libnvme APIs to pass flag
- gen_tls_key, check_tls_key
(I don't think nvme_tls_key does key conversions)

- add command line flag to generic parsing into nvme_fabrics_cfg
- handles connect, connect-all, discover, discover-all

- man pages, shell completions, etc.

It may be possible to keep this to just in libnvme and switch on an environment variable, but that sounds like just as big of a support nightmare as remembering to pass an option to every command.

If we don't have a resolution by then, maybe this is something we can resolve at ALPSS.

[1] Side note; with nvme_fabrics_cfg _not_ being allocated through libnvme, I don't see how the various additions there can be ABI safe. It looks to me like this struct alone broke ABI in libnvme 1.4, 1.8, 1.9, and 1.11.

- Chris

#9 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 77 lines · Chris Leech
Quoted 33 lines · Hannes Reinecke

...
But: this is an incompatible change. Once we integrate this patch the keyring will end up with a _different_ derived PSK than it would without that patch. So the _same_ PSK (in interchange format) will result in different PSKs in the keyring, causing the handshake to fail if the other side is not aware of that.

Quoted 9 lines
Quoted 8 lines

_However_: PSKs generated with after applying this patch will be different than those prior to this patch.
Consequently there will be interop issues with existing implementations (which will use the original encoding).

I guess we would need to wait for the target implementations to be fixed or introduce a flag switching to the old / compat implementation to avoid interop issues.

Quoted 7 lines

Yes, it is a breaking change for compatibility with other out-of-spec implementations. I'm hesitant to carry a flag for that, but it wouldn't be too much trouble to implement.

Just don't touch your keyfile? The tls-key import/export commands are operating on the final derived TLS PSKs right?

That might be the way out, and in fact what we should be doing. However, the fact still remains: after this patch has been applied one _cannot_ generate new PSKs until the other side has been updated, too.
For linux it's easy, we can just require 'version xyz' from libnvme and the issue is solved. But for other implementations? There are IHVs out there who already shipped their products with TLS enablement, and they need to be fixed, too.
And their timeline is vastly longer than ours.

Quoted 6 lines

So to avoid us having to synchronize against all of the others I think it might be easier to add a 'compat' flag of sorts to generate PSKs with the 'original' derivation algorithm, and then increase the libnvme version number once it's in.
Then we can point the IHVs to that number so that they reference that version once their firmware is updated.

I've spent a bit of time on this, and while I don't have patches ready to share just yet I do think it's worth commenting on the scope of the change to keep the existing functionality behind a cli switch.

libnvme:
- keeping both sets of key derivation functions (this is not bad)
- need to expose that functionality, so new exported APIs
(I was thinking something like a flag field over a legacy-only function)
- #define NVME_TLS_PSK_LEGACY (1u << 0)
- char *nvme_generate_tls_key_identity_2(..., unsigned int flags)
- long nvme_insert_tls_key_versioned_2(..., unsigned int flags)

- support paths that load keys through nvme_fabrics_cfg [1] (for nvme-cli fabrics commands that take --tls_key)
- add some sort of flag there, like nvme_fabrics_cfg.tls_legacy
- add to merge_config, nvmf_update_config, build_options, supported_options, etc.

- the JSON schema/code would need to support this for persistent configurations

nvme-cli:
- add a command line flag to keyring command functions
- convert to use of new libnvme APIs to pass flag
- gen_tls_key, check_tls_key
(I don't think nvme_tls_key does key conversions)

- add command line flag to generic parsing into nvme_fabrics_cfg
- handles connect, connect-all, discover, discover-all

- man pages, shell completions, etc.

It may be possible to keep this to just in libnvme and switch on an environment variable, but that sounds like just as big of a support nightmare as remembering to pass an option to every command.

Quoted 8 lines

If we don't have a resolution by then, maybe this is something we can resolve at ALPSS.

[1] Side note; with nvme_fabrics_cfg _not_ being allocated through libnvme, I don't see how the various additions there can be ABI safe. It looks to me like this struct alone broke ABI in libnvme 1.4, 1.8, 1.9, and 1.11.

My thoughts are slightly different.
I do agree that we need to change the default from nvme-cli to use the
correct key derivation.
But for compat we should be using the 'tls-key' function, and add a 'compat' flag there. That way we don't need to modify 'gen-tls-key' or 'check-tls-key', and have a single place to handle compat issues.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#10 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

In reply to Chris Leech
Quoted 7 lines · Chris Leech

If we don't have a resolution by then, maybe this is something we can resolve at ALPSS.

[1] Side note; with nvme_fabrics_cfg _not_ being allocated through libnvme, I don't see how the various additions there can be ABI safe. It looks to me like this struct alone broke ABI in libnvme 1.4, 1.8, 1.9, and 1.11.

I know and I was hopping it wouldn't cause troubles... nvme_fabrics_cfg is a bit of a problem. I was planing to address this for v2.

#11 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 11 lines · Daniel Wagner
Quoted 7 lines · Chris Leech

If we don't have a resolution by then, maybe this is something we can resolve at ALPSS.

[1] Side note; with nvme_fabrics_cfg _not_ being allocated through libnvme, I don't see how the various additions there can be ABI safe. It looks to me like this struct alone broke ABI in libnvme 1.4, 1.8, 1.9, and 1.11.

I know and I was hopping it wouldn't cause troubles... nvme_fabrics_cfg is a bit of a problem. I was planing to address this for v2.

In a similar vein: shouldn't we have a session at ALPSS around libnvme v2?
Or, if not a session, maybe a beer topic?

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#12 lore raw

Re: [PATCH 1/1] libnvme: TLS PSK derivation fixes

Quoted 10 lines · Hannes Reinecke
Quoted 9 lines · Daniel Wagner
Quoted 7 lines · Chris Leech

If we don't have a resolution by then, maybe this is something we can resolve at ALPSS.

[1] Side note; with nvme_fabrics_cfg _not_ being allocated through libnvme, I don't see how the various additions there can be ABI safe. It looks to me like this struct alone broke ABI in libnvme 1.4, 1.8, 1.9, and 1.11.

Quoted 5 lines
Quoted 2 lines

I know and I was hopping it wouldn't cause troubles... nvme_fabrics_cfg is a bit of a problem. I was planing to address this for v2.

In a similar vein: shouldn't we have a session at ALPSS around libnvme v2?

Yes, both Chris and I have proposed a talk to discuss NVMe/TLS at ALPSS.

Quoted 1 line

Or, if not a session, maybe a beer topic?

Sure, you can drink a beer when we talk about it. ;-)

/John

Quoted 3 lines

Cheers,

Hannes

#13 lore raw
In reply to Chris Leech
Quoted 16 lines · Chris Leech

I was attempting to debug connecting the Linux driver / libnvme / ktls-utils host stack to the SPDK nvmf_tgt over TLS, and ran into some issues.

The TLS connection fails to complete a handshake because the TLS PSKs are different. The NVMe/TCP specified key derivation steps from the configured interchange format, to a retained PSK and finally the TLS PSK, is implemented incompatibly in libnvme and SPDK. After some investigation, I believe the SPDK implementation to be correct and am providing a libnvme patch to match it. With libnvme modified, I see the TLS handshake complete in tlshd.

(Note that this was tested using the obsolete "version 0" PSK Identity and TLS PSK derivation from the TCP transport 1.0 specification, as SPDK has not been updated with the "version 1" changes)

Quoted 8 lines

The NVMe/TCP host driver then quickly fails when SPDK sends a TLS "New Session Ticket" message before ICResp.

While possibly pointless due to the transport specification prohibition on session resumption and 0-RTT data, I don't think this is necessarily wrong and the host driver should be able to safely ignore it and continue.

Sigh. The neverending discussion.
Originally I didn't implement key update for NVMe as neither gnutls nor the kernel supported it.
Turns out that key update was implemented last year for ktls, and gnutls has support functions (gnutls_session_key_update()) for this, too. So we _could_ start looking into it if nvme wouldn't be using the ->read_sock() interface.
This interface just works on skbs, and as such we can't easily get hold on the TLS alert data required for figuring out if a New Session Ticket message had been sent.

I do have a patchset for converting that into a recvmsg() based implementation, but that got shelved as we didn't have an efficient iterator for data hashing. With the crc32 update from Eric Biggers we now should have an iterator for crc32 data hashing, so I guess I could revisit that.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#14 lore raw
Quoted 10 lines · Hannes Reinecke
Quoted 9 lines · Chris Leech

...
The NVMe/TCP host driver then quickly fails when SPDK sends a TLS "New Session Ticket" message before ICResp.

While possibly pointless due to the transport specification prohibition on session resumption and 0-RTT data, I don't think this is necessarily wrong and the host driver should be able to safely ignore it and continue.

Quoted 10 lines

Sigh. The neverending discussion.
Originally I didn't implement key update for NVMe as neither gnutls nor the kernel supported it.
Turns out that key update was implemented last year for ktls, and gnutls has support functions (gnutls_session_key_update()) for this, too. So we _could_ start looking into it if nvme wouldn't be using the ->read_sock() interface.
This interface just works on skbs, and as such we can't easily get hold on the TLS alert data required for figuring out if a New Session Ticket message had been sent.

I think this is a bit different, as it's NewSessionTicket and not KeyUpdate. NewSessionTicket seems to provide an age-limited PSK for session resumption, allowing additional connections or re-connections to bypass an expensive certificate handshake. We're only using retained PSKs and prohibited from session resumption, so while spdk probably shouldn't be sending NewSessionTicket it should also safe for the host to just filter it out of the data stream and never use it.

- Chris

#15 lore raw
Quoted 22 lines · Chris Leech
Quoted 10 lines · Hannes Reinecke
Quoted 9 lines · Chris Leech

...
The NVMe/TCP host driver then quickly fails when SPDK sends a TLS "New Session Ticket" message before ICResp.

While possibly pointless due to the transport specification prohibition on session resumption and 0-RTT data, I don't think this is necessarily wrong and the host driver should be able to safely ignore it and continue.

Quoted 10 lines

Sigh. The neverending discussion.
Originally I didn't implement key update for NVMe as neither gnutls nor the kernel supported it.
Turns out that key update was implemented last year for ktls, and gnutls has support functions (gnutls_session_key_update()) for this, too. So we _could_ start looking into it if nvme wouldn't be using the ->read_sock() interface.
This interface just works on skbs, and as such we can't easily get hold on the TLS alert data required for figuring out if a New Session Ticket message had been sent.

Quoted 8 lines

I think this is a bit different, as it's NewSessionTicket and not KeyUpdate. NewSessionTicket seems to provide an age-limited PSK for session resumption, allowing additional connections or re-connections to bypass an expensive certificate handshake. We're only using retained PSKs and prohibited from session resumption, so while spdk probably shouldn't be sending NewSessionTicket it should also safe for the host to just filter it out of the data stream and never use it.

Ah, the NewSessionTicket thingie. Yes, that's completely pointless for NVMe, as we _never_ re-use tickets for session re-establishment. We only ever have one session, so the use-case of several short-lived sessions doesn't apply here.

But with the current ->read_sock based receive workflow we cannot filter out TLS NewSession messages from the NVMe layer as we don't have access to the TCP stream itself. So either we switch to a recvmsg() based workflow or we need to modify the read_sock() implementation in net/tls to filter out the NewSession message there.

Personally I'd rather switch over to recvmsg() as I never liked the read_sock() interface anyway.
But that's my personal opinion; others may have different ideas.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#16 lore raw
Quoted 4 lines · Hannes Reinecke

Ah, the NewSessionTicket thingie. Yes, that's completely pointless for NVMe, as we _never_ re-use tickets for session re-establishment. We only ever have one session, so the use-case of several short-lived sessions doesn't apply here.

Not following the discussion, can you either nacking this patch or give a reviewed tag? Thanks.

#17 lore raw
Quoted 9 lines · Daniel Wagner
Quoted 4 lines · Hannes Reinecke

Ah, the NewSessionTicket thingie. Yes, that's completely pointless for NVMe, as we _never_ re-use tickets for session re-establishment. We only ever have one session, so the use-case of several short-lived sessions doesn't apply here.

Not following the discussion, can you either nacking this patch or give a reviewed tag? Thanks.

Hi Daniel,

I believe Hannes has given a reviewed tag on the libvnme patch. The discussion on the cover letter is about future work on the kernel side.

If it would help I can capture that tag and then submit this as a github PR. Actually, I think I might be missing an explicit NULL assignment on the declaration of *str to keep _cleanup_free_ safe?

-Chris

#18 lore raw

Hi Chris,

Quoted 3 lines · Chris Leech

I believe Hannes has given a reviewed tag on the libvnme patch. The discussion on the cover letter is about future work on the kernel side.

Ah, thanks. Trimming emails would help.

Quoted 2 lines

If it would help I can capture that tag and then submit this as a github PR.

The PR has the advantage that a CI build gets triggered. Submissions on the mailing list are usually in good shape. Submissions via GitHub vary quite a bit. I’m fine either way. :)

Quoted 2 lines

Actually, I think I might be missing an explicit NULL assignment on the declaration of *str to keep _cleanup_free_ safe?

For _cleanup_free_ to work, the variable needs to be initialized (either NULL or a valid pointer) before the block scope is left. Otherwise the free sees a random stack value.

Thanks,
Daniel

#19 lore raw
In reply to Chris Leech
Quoted 11 lines · Chris Leech

I was attempting to debug connecting the Linux driver / libnvme / ktls-utils host stack to the SPDK nvmf_tgt over TLS, and ran into some issues.

The TLS connection fails to complete a handshake because the TLS PSKs are different. The NVMe/TCP specified key derivation steps from the configured interchange format, to a retained PSK and finally the TLS PSK, is implemented incompatibly in libnvme and SPDK. After some investigation, I believe the SPDK implementation to be correct and am providing a libnvme patch to match it. With libnvme modified, I see the TLS handshake complete in tlshd.

Ug, the kernel has key derivation code for secure channel concatenation and it has the same issue.

RFC patchs to follow, only tested with a quick blktests nvme/063 currently.

- Chris

#20 lore raw

RFC PATCH 1/2 crypto: hkdf: add hkdf_expand_label()

In reply to Chris Leech

Provide an implementation of RFC 8446 (TLS 1.3) HKDF-Expand-Label

---
 crypto/hkdf.c         | 55 +++++++++++++++++++++++++++++++++++++++++++
 include/crypto/hkdf.h |  4 ++++
 2 files changed, 59 insertions(+)
crypto/hkdf.c +55 -0
index 82d1b32ca6ce4..465bad6e6c93e 100644
@@ -11,6 +11,7 @@
 #include <crypto/sha2.h>
 #include <crypto/hkdf.h>
 #include <linux/module.h>
+#include <linux/unaligned.h>
 
 /*
  * HKDF consists of two steps:
@@ -129,6 +130,60 @@ int hkdf_expand(struct crypto_shash *hmac_tfm,
 }
 EXPORT_SYMBOL_GPL(hkdf_expand);
 
+/**
+ * hkdf_expand_label - HKDF-Expand-Label (RFC 8846 section 7.1)
+ * @hmac_tfm: hash context keyed with pseudorandom key
+ * @label: ASCII label without "tls13 " prefix
+ * @label_len: length of @label
+ * @context: context bytes
+ * @contextlen: length of @context
+ * @okm: output keying material
+ * @okmlen: length of @okm
+ *
+ * Build the TLS 1.3 HkdfLabel structure and invoke hkdf_expand().
+ *
+ * Returns 0 on success with output keying material stored in @okm,
+ * or a negative errno value otherwise.
+ */
+int hkdf_expand_label(struct crypto_shash *hmac_tfm,
+		const u8 *label, unsigned int labellen,
+		const u8 *context, unsigned int contextlen,
+		u8 *okm, unsigned int okmlen)
+{
+	int err;
+	u8 *info;
+	unsigned int infolen;
+	static const char tls13_prefix[] = "tls13 ";
+	unsigned int prefixlen = sizeof(tls13_prefix) - 1; /* exclude NUL */
+
+	if (WARN_ON(labellen > (255 - prefixlen)))
+		return -EINVAL;
+	if (WARN_ON(contextlen > 255))
+		return -EINVAL;
+
+	infolen = 2 + (1 + prefixlen + labellen) + (1 + contextlen);
+	info = kzalloc(infolen, GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	/* HkdfLabel.Length */
+	put_unaligned_be16(okmlen, info);
+
+	/* HkdfLabel.Label */
+	info[2] = prefixlen + labellen;
+	memcpy(info + 3, tls13_prefix, prefixlen);
+	memcpy(info + 3 + prefixlen, label, labellen);
+
+	/* HkdfLabel.Context */
+	info[3 + prefixlen + labellen] = contextlen;
+	memcpy(info + 4 + prefixlen + labellen, context, contextlen);
+
+	err = hkdf_expand(hmac_tfm, info, infolen, okm, okmlen);
+	kfree_sensitive(info);
+	return err;
+}
+EXPORT_SYMBOL_GPL(hkdf_expand_label);
+
 struct hkdf_testvec {
 	const char *test;
 	const u8 *ikm;
include/crypto/hkdf.h +4 -0
index 6a9678f508f5d..5e75d17a58abe 100644
@@ -17,4 +17,8 @@ int hkdf_extract(struct crypto_shash *hmac_tfm, const u8 *ikm,
 int hkdf_expand(struct crypto_shash *hmac_tfm,
 		const u8 *info, unsigned int infolen,
 		u8 *okm, unsigned int okmlen);
+int hkdf_expand_label(struct crypto_shash *hmac_tfm,
+		const u8 *label, unsigned int labellen,
+		const u8 *context, unsigned int contextlen,
+		u8 *okm, unsigned int okmlen);
 #endif
Signature · 22 lines
2.49.0


From 17e645b802bd61b4f3182195c15935e2fa1155ad Mon Sep 17 00:00:00 2001
From: Chris Leech <[email protected]>
Date: Tue, 12 Aug 2025 12:41:08 -0700
Subject: [RFC PATCH 2/2] nvme-auth: use hkdf_expand_label()

When generating keying material during an authentication transaction
(secure channel concatenation), the HKDF-Expand-Label function is part
of the specified key derivation process.

The current open-coded implementation misses the length prefix
requirements on the HkdfLabel label and context variable-length vectors
(RFC 8446 Section 3.4).

Instead, use the hkdf_expand_label() function.

Signed-off-by: Chris Leech <[email protected]>
---
 drivers/nvme/common/auth.c | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)
drivers/nvme/common/auth.c +14 -20
index 91e273b89fea3..bfa6f0a82d3e9 100644
@@ -715,10 +715,10 @@ int nvme_auth_derive_tls_psk(int hmac_id, u8 *psk, size_t psk_len,
 {
 	struct crypto_shash *hmac_tfm;
 	const char *hmac_name;
-	const char *psk_prefix = "tls13 nvme-tls-psk";
 	static const char default_salt[HKDF_MAX_HASHLEN];
-	size_t info_len, prk_len;
-	char *info;
+	size_t prk_len;
+	const char *label = "nvme-tls-psk";
+	const char *ctx;
 	unsigned char *prk, *tls_key;
 	int ret;
 
@@ -758,36 +758,30 @@ int nvme_auth_derive_tls_psk(int hmac_id, u8 *psk, size_t psk_len,
 	if (ret)
 		goto out_free_prk;
 
-	/*
-	 * 2 additional bytes for the length field from HDKF-Expand-Label,
-	 * 2 additional bytes for the HMAC ID, and one byte for the space
-	 * separator.
-	 */
-	info_len = strlen(psk_digest) + strlen(psk_prefix) + 5;
-	info = kzalloc(info_len + 1, GFP_KERNEL);
-	if (!info) {
+	/* Context is ASCII: "<hash-id-two-digits> <base64-digest>" */
+	ctx = kasprintf(GFP_KERNEL, "%02d %s", hmac_id, psk_digest);
+	if (!ctx) {
 		ret = -ENOMEM;
 		goto out_free_prk;
 	}
 
-	put_unaligned_be16(psk_len, info);
-	memcpy(info + 2, psk_prefix, strlen(psk_prefix));
-	sprintf(info + 2 + strlen(psk_prefix), "%02d %s", hmac_id, psk_digest);
-
 	tls_key = kzalloc(psk_len, GFP_KERNEL);
 	if (!tls_key) {
 		ret = -ENOMEM;
-		goto out_free_info;
+		goto out_free_ctx;
 	}
-	ret = hkdf_expand(hmac_tfm, info, info_len, tls_key, psk_len);
+	ret = hkdf_expand_label(hmac_tfm,
+				label, strlen(label),
+				ctx, strlen(ctx),
+				tls_key, psk_len);
 	if (ret) {
 		kfree(tls_key);
-		goto out_free_info;
+		goto out_free_ctx;
 	}
 	*ret_psk = tls_key;
 
-out_free_info:
-	kfree(info);
+out_free_ctx:
+	kfree(ctx);
 out_free_prk:
 	kfree(prk);
 out_free_shash:
Signature · 1 line
2.49.0
#21 lore raw

Re: [RFC PATCH 1/2] crypto: hkdf: add hkdf_expand_label()

Quoted 2 lines · Chris Leech

Provide an implementation of RFC 8446 (TLS 1.3) HKDF-Expand-Label

Looks good, but patches are garbled.
Please re-post as a separate patchset with a cover letter.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich