tlshd: Add runtime check for ML-DSA support in gnutls

From: Chuck Lever <[email protected]>

Fedora 42 ships gnutls 3.8.11, which defines GNUTLS_PK_MLDSA* and GNUTLS_SIGN_MLDSA* enum constants in its headers and can parse ML-DSA certificates and import ML-DSA private keys. However, it does not register SHAKE-256 as an available digest
algorithm: gnutls_hash_get_len(GNUTLS_DIG_SHAKE_256) returns 0 and SHAKE-256 is absent from gnutls_digest_list(). Because the ML-DSA sign algorithm table entries specify GNUTLS_DIG_SHAKE_256 as their hash, all ML-DSA signing operations fail at the hash step during TLS handshakes.

RHEL 10's gnutls 3.8.10 carries downstream patches that wire up SHAKE-256 in the digest framework, so ML-DSA works there. The existing compile-time configure check (AC_COMPILE_IFELSE on the GNUTLS_SIGN_MLDSA65 constant) cannot distinguish these two
cases: both define the constant, but only one has a functional implementation.

The symptom on Fedora 42 is that tlshd's server-side retrieve_key callback selects the ML-DSA certificate (because the client's gnutls also advertises ML-DSA signature algorithms in its ClientHello), gnutls fails at cert_select_sign_algorithm for every cipher suite, and the handshake fails with "No supported cipher suites have been found. (-87)". The RSA fallback never runs because gnutls retries the callback for each cipher suite and gets the same unusable ML-DSA certificate each time.

Probe gnutls_hash_get_len(GNUTLS_DIG_SHAKE_256) at certificate load time to detect whether the hash prerequisite for ML-DSA is present. When it returns zero, tlshd_cert_check_pk_alg() returns false, disabling the post-quantum certificate so the RSA certificate is used instead.

---
 src/tlshd/config.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

index ce48a4d17fc2..1ed94a7776ad 100644
@@ -41,6 +41,7 @@
 
 #include <gnutls/gnutls.h>
 #include <gnutls/abstract.h>
+#include <gnutls/crypto.h>
 
 #include <netlink/netlink.h>
 

@@ -384,6 +385,7 @@ static bool tlshd_cert_check_pk_alg(gnutls_datum_t *data,
 {
 	gnutls_x509_crt_t cert;
 	gnutls_pk_algorithm_t pk_alg;
+	bool result = false;
 	int ret;
 
 	ret = gnutls_x509_crt_init(&cert);

@@ -391,10 +393,8 @@ static bool tlshd_cert_check_pk_alg(gnutls_datum_t *data,
 		return false;
 
 	ret = gnutls_x509_crt_import(cert, data, GNUTLS_X509_FMT_PEM);
-	if (ret < 0) {
-		gnutls_x509_crt_deinit(cert);
-		return false;
-	}
+	if (ret < 0)
+		goto out;
 
 	pk_alg = gnutls_x509_crt_get_pk_algorithm(cert, NULL);
 	tlshd_log_debug("%s: certificate pk algorithm %s", __func__,

@@ -403,15 +403,23 @@ static bool tlshd_cert_check_pk_alg(gnutls_datum_t *data,
 	case GNUTLS_PK_MLDSA44:
 	case GNUTLS_PK_MLDSA65:
 	case GNUTLS_PK_MLDSA87:
+		if (!gnutls_hash_get_len(GNUTLS_DIG_SHAKE_256)) {
+			tlshd_log_notice("%s: gnutls lacks SHAKE-256 support, "
+					 "disabling %s",
+					 __func__,
+					 gnutls_pk_algorithm_get_name(pk_alg));
+			goto out;
+		}
 		*pkalg = pk_alg;
+		result = true;
 		break;
 	default:
-		gnutls_x509_crt_deinit(cert);
-		return false;
+		break;
 	}
 
+out:
 	gnutls_x509_crt_deinit(cert);
-	return true;
+	return result;
 }
 #else
 static bool tlshd_cert_check_pk_alg(__attribute__ ((unused)) gnutls_datum_t *data,

2.53.0