GHASH library

index f4cd6c29b4d3..ad5edf9366ac 100644
@@ -14,13 +14,13 @@
  *
  * For more details, see the paper: "Length-preserving encryption with HCTR2"
  * (https://eprint.iacr.org/2021/1441.pdf)
  */
 
+#include <crypto/gf128hash.h>
 #include <crypto/internal/cipher.h>
 #include <crypto/internal/skcipher.h>
-#include <crypto/polyval.h>
 #include <crypto/scatterwalk.h>
 #include <linux/module.h>
 
 #define BLOCKCIPHER_BLOCK_SIZE		16
 

similarity index 94%
rename from include/crypto/polyval.h
rename to include/crypto/gf128hash.h
index b28b8ef11353..5ffa86f5c13f 100644
@@ -1,14 +1,14 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
- * POLYVAL library API
+ * GF(2^128) polynomial hashing: GHASH and POLYVAL
  *
  * Copyright 2025 Google LLC
  */
 
-#ifndef _CRYPTO_POLYVAL_H
-#define _CRYPTO_POLYVAL_H
+#ifndef _CRYPTO_GF128HASH_H
+#define _CRYPTO_GF128HASH_H
 
 #include <linux/string.h>
 #include <linux/types.h>
 
 #define POLYVAL_BLOCK_SIZE	16

@@ -42,24 +42,24 @@ struct polyval_elem {
  *
  * By H^i we mean H^(i-1) * H * x^-128, with base case H^1 = H.  I.e. the
  * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128.
  */
 struct polyval_key {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 #ifdef CONFIG_ARM64
 	/** @h_powers: Powers of the hash key H^8 through H^1 */
 	struct polyval_elem h_powers[8];
 #elif defined(CONFIG_X86)
 	/** @h_powers: Powers of the hash key H^8 through H^1 */
 	struct polyval_elem h_powers[8];
 #else
 #error "Unhandled arch"
 #endif
-#else /* CONFIG_CRYPTO_LIB_POLYVAL_ARCH */
+#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
 	/** @h: The hash key H */
 	struct polyval_elem h;
-#endif /* !CONFIG_CRYPTO_LIB_POLYVAL_ARCH */
+#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
 };
 
 /**
  * struct polyval_ctx - Context for computing a POLYVAL value
  * @key: Pointer to the prepared POLYVAL key.  The user of the API is

@@ -82,11 +82,11 @@ struct polyval_ctx {
  * copy, or it may involve precomputing powers of the key, depending on the
  * platform's POLYVAL implementation.
  *
  * Context: Any context.
  */
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE]);
 
 #else
 static inline void polyval_preparekey(struct polyval_key *key,

@@ -185,6 +185,6 @@ static inline void polyval(const struct polyval_key *key,
 	polyval_init(&ctx, key);
 	polyval_update(&ctx, data, len);
 	polyval_final(&ctx, out);
 }
 
-#endif /* _CRYPTO_POLYVAL_H */
+#endif /* _CRYPTO_GF128HASH_H */

similarity index 95%
rename from lib/crypto/arm64/polyval.h
rename to lib/crypto/arm64/gf128hash.h
index a39763395e9b..c1012007adcf 100644
@@ -70,11 +70,11 @@ static void polyval_blocks_arch(struct polyval_elem *acc,
 		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
 				       data, nblocks);
 	}
 }
 
-#define polyval_mod_init_arch polyval_mod_init_arch
-static void polyval_mod_init_arch(void)
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
 {
 	if (cpu_have_named_feature(PMULL))
 		static_branch_enable(&have_pmull);
 }

similarity index 94%
rename from lib/crypto/polyval.c
rename to lib/crypto/gf128hash.c
index 5796275f574a..8bb848bf26b7 100644
@@ -1,13 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * POLYVAL library functions
+ * GF(2^128) polynomial hashing: GHASH and POLYVAL
  *
  * Copyright 2025 Google LLC
  */
 
-#include <crypto/polyval.h>
+#include <crypto/gf128hash.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/unaligned.h>
 

@@ -216,12 +216,12 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
 		data += POLYVAL_BLOCK_SIZE;
 	} while (--nblocks);
 }
 
 /* Include the arch-optimized implementation of POLYVAL, if one is available. */
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
-#include "polyval.h" /* $(SRCARCH)/polyval.h */
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
 	polyval_preparekey_arch(key, raw_key);
 }

@@ -236,21 +236,21 @@ EXPORT_SYMBOL_GPL(polyval_preparekey);
  * code is needed to pass the appropriate key argument.
  */
 
 static void polyval_mul(struct polyval_ctx *ctx)
 {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 	polyval_mul_arch(&ctx->acc, ctx->key);
 #else
 	polyval_mul_generic(&ctx->acc, &ctx->key->h);
 #endif
 }
 
 static void polyval_blocks(struct polyval_ctx *ctx,
 			   const u8 *data, size_t nblocks)
 {
-#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH
+#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 	polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
 #else
 	polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);
 #endif
 }

@@ -287,21 +287,21 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE])
 	memcpy(out, &ctx->acc, POLYVAL_BLOCK_SIZE);
 	memzero_explicit(ctx, sizeof(*ctx));
 }
 EXPORT_SYMBOL_GPL(polyval_final);
 
-#ifdef polyval_mod_init_arch
-static int __init polyval_mod_init(void)
+#ifdef gf128hash_mod_init_arch
+static int __init gf128hash_mod_init(void)
 {
-	polyval_mod_init_arch();
+	gf128hash_mod_init_arch();
 	return 0;
 }
-subsys_initcall(polyval_mod_init);
+subsys_initcall(gf128hash_mod_init);
 
-static void __exit polyval_mod_exit(void)
+static void __exit gf128hash_mod_exit(void)
 {
 }
-module_exit(polyval_mod_exit);
+module_exit(gf128hash_mod_exit);
 #endif
 
-MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function");
+MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL");
 MODULE_LICENSE("GPL");

index f47f41a39a41..d1f53a690ab8 100644
@@ -1,10 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * Copyright 2025 Google LLC
  */
-#include <crypto/polyval.h>
+#include <crypto/gf128hash.h>
 #include "polyval-testvecs.h"
 
 /*
  * A fixed key used when presenting POLYVAL as an unkeyed hash function in order
  * to reuse hash-test-template.h.  At the beginning of the test suite, this is

similarity index 95%
rename from lib/crypto/x86/polyval.h
rename to lib/crypto/x86/gf128hash.h
index ef8797521420..fe506cf6431b 100644
@@ -72,12 +72,12 @@ static void polyval_blocks_arch(struct polyval_elem *acc,
 		polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
 				       data, nblocks);
 	}
 }
 
-#define polyval_mod_init_arch polyval_mod_init_arch
-static void polyval_mod_init_arch(void)
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
 {
 	if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) &&
 	    boot_cpu_has(X86_FEATURE_AVX))
 		static_branch_enable(&have_pclmul_avx);
 }

index 5ffa86f5c13f..1052041e3499 100644
@@ -42,24 +42,18 @@ struct polyval_elem {
  *
  * By H^i we mean H^(i-1) * H * x^-128, with base case H^1 = H.  I.e. the
  * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128.
  */
 struct polyval_key {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
-#ifdef CONFIG_ARM64
-	/** @h_powers: Powers of the hash key H^8 through H^1 */
-	struct polyval_elem h_powers[8];
-#elif defined(CONFIG_X86)
+#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
+	(defined(CONFIG_ARM64) || defined(CONFIG_X86))
 	/** @h_powers: Powers of the hash key H^8 through H^1 */
 	struct polyval_elem h_powers[8];
 #else
-#error "Unhandled arch"
-#endif
-#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
 	/** @h: The hash key H */
 	struct polyval_elem h;
-#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */
+#endif
 };
 
 /**
  * struct polyval_ctx - Context for computing a POLYVAL value
  * @key: Pointer to the prepared POLYVAL key.  The user of the API is

@@ -82,23 +76,13 @@ struct polyval_ctx {
  * copy, or it may involve precomputing powers of the key, depending on the
  * platform's POLYVAL implementation.
  *
  * Context: Any context.
  */
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE]);
 
-#else
-static inline void polyval_preparekey(struct polyval_key *key,
-				      const u8 raw_key[POLYVAL_BLOCK_SIZE])
-{
-	/* Just a simple copy, so inline it. */
-	memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
-}
-#endif
-
 /**
  * polyval_init() - Initialize a POLYVAL context for a new message
  * @ctx: The context to initialize
  * @key: The key to use.  Note that a pointer to the key is saved in the
  *	 context, so the key must live at least as long as the context.

index c1012007adcf..796c36804dda 100644
@@ -15,10 +15,11 @@ asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
 				  const struct polyval_elem *b);
 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
 				     const struct polyval_key *key,
 				     const u8 *data, size_t nblocks);
 
+#define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,
 				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
 	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
 	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);

@@ -38,10 +39,11 @@ static void polyval_preparekey_arch(struct polyval_key *key,
 					    &key->h_powers[NUM_H_POWERS - 1]);
 		}
 	}
 }
 
+#define polyval_mul_arch polyval_mul_arch
 static void polyval_mul_arch(struct polyval_elem *acc,
 			     const struct polyval_key *key)
 {
 	if (static_branch_likely(&have_pmull) && may_use_simd()) {
 		scoped_ksimd()

@@ -49,10 +51,11 @@ static void polyval_mul_arch(struct polyval_elem *acc,
 	} else {
 		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
 	}
 }
 
+#define polyval_blocks_arch polyval_blocks_arch
 static void polyval_blocks_arch(struct polyval_elem *acc,
 				const struct polyval_key *key,
 				const u8 *data, size_t nblocks)
 {
 	if (static_branch_likely(&have_pmull) && may_use_simd()) {

index 8bb848bf26b7..05f44a9193f7 100644
@@ -215,20 +215,24 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key,
 		polyval_mul_generic(acc, key);
 		data += POLYVAL_BLOCK_SIZE;
 	} while (--nblocks);
 }
 
-/* Include the arch-optimized implementation of POLYVAL, if one is available. */
 #ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
 #include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */
+#endif
+
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
+#ifdef polyval_preparekey_arch
 	polyval_preparekey_arch(key, raw_key);
+#else
+	memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE);
+#endif
 }
 EXPORT_SYMBOL_GPL(polyval_preparekey);
-#endif /* Else, polyval_preparekey() is an inline function. */
 
 /*
  * polyval_mul_generic() and polyval_blocks_generic() take the key as a
  * polyval_elem rather than a polyval_key, so that arch-optimized
  * implementations with a different key format can use it as a fallback (if they

@@ -236,21 +240,25 @@ EXPORT_SYMBOL_GPL(polyval_preparekey);
  * code is needed to pass the appropriate key argument.
  */
 
 static void polyval_mul(struct polyval_ctx *ctx)
 {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#ifdef polyval_mul_arch
 	polyval_mul_arch(&ctx->acc, ctx->key);
+#elif defined(polyval_blocks_arch)
+	static const u8 zeroes[POLYVAL_BLOCK_SIZE];
+
+	polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1);
 #else
 	polyval_mul_generic(&ctx->acc, &ctx->key->h);
 #endif
 }
 
 static void polyval_blocks(struct polyval_ctx *ctx,
 			   const u8 *data, size_t nblocks)
 {
-#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH
+#ifdef polyval_blocks_arch
 	polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks);
 #else
 	polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks);
 #endif
 }

index fe506cf6431b..adf6147ea677 100644
@@ -15,10 +15,11 @@ asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a,
 				       const struct polyval_elem *b);
 asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc,
 					  const struct polyval_key *key,
 					  const u8 *data, size_t nblocks);
 
+#define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,
 				    const u8 raw_key[POLYVAL_BLOCK_SIZE])
 {
 	static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
 	memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);

@@ -38,10 +39,11 @@ static void polyval_preparekey_arch(struct polyval_key *key,
 					    &key->h_powers[NUM_H_POWERS - 1]);
 		}
 	}
 }
 
+#define polyval_mul_arch polyval_mul_arch
 static void polyval_mul_arch(struct polyval_elem *acc,
 			     const struct polyval_key *key)
 {
 	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
 		kernel_fpu_begin();

@@ -50,10 +52,11 @@ static void polyval_mul_arch(struct polyval_elem *acc,
 	} else {
 		polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]);
 	}
 }
 
+#define polyval_blocks_arch polyval_blocks_arch
 static void polyval_blocks_arch(struct polyval_elem *acc,
 				const struct polyval_key *key,
 				const u8 *data, size_t nblocks)
 {
 	if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {

index 1052041e3499..5090fbaa87f8 100644
@@ -9,10 +9,12 @@
 #define _CRYPTO_GF128HASH_H
 
 #include <linux/string.h>
 #include <linux/types.h>
 
+#define GHASH_BLOCK_SIZE	16
+#define GHASH_DIGEST_SIZE	16
 #define POLYVAL_BLOCK_SIZE	16
 #define POLYVAL_DIGEST_SIZE	16
 
 /**
  * struct polyval_elem - An element of the POLYVAL finite field

@@ -31,10 +33,20 @@ struct polyval_elem {
 			__le64 hi;
 		};
 	};
 };
 
+/**
+ * struct ghash_key - Prepared key for GHASH
+ *
+ * Use ghash_preparekey() to initialize this.
+ */
+struct ghash_key {
+	/** @h: The hash key H, in POLYVAL format */
+	struct polyval_elem h;
+};
+
 /**
  * struct polyval_key - Prepared key for POLYVAL
  *
  * This may contain just the raw key H, or it may contain precomputed key
  * powers, depending on the platform's POLYVAL implementation.  Use

@@ -52,10 +64,24 @@ struct polyval_key {
 	/** @h: The hash key H */
 	struct polyval_elem h;
 #endif
 };
 
+/**
+ * struct ghash_ctx - Context for computing a GHASH value
+ * @key: Pointer to the prepared GHASH key.  The user of the API is
+ *	 responsible for ensuring that the key lives as long as the context.
+ * @acc: The accumulator.  It is stored in POLYVAL format rather than GHASH
+ *	 format, since most implementations want it in POLYVAL format.
+ * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE
+ */
+struct ghash_ctx {
+	const struct ghash_key *key;
+	struct polyval_elem acc;
+	size_t partial;
+};
+
 /**
  * struct polyval_ctx - Context for computing a POLYVAL value
  * @key: Pointer to the prepared POLYVAL key.  The user of the API is
  *	 responsible for ensuring that the key lives as long as the context.
  * @acc: The accumulator

@@ -65,10 +91,22 @@ struct polyval_ctx {
 	const struct polyval_key *key;
 	struct polyval_elem acc;
 	size_t partial;
 };
 
+/**
+ * ghash_preparekey() - Prepare a GHASH key
+ * @key: (output) The key structure to initialize
+ * @raw_key: The raw hash key
+ *
+ * Initialize a GHASH key structure from a raw key.
+ *
+ * Context: Any context.
+ */
+void ghash_preparekey(struct ghash_key *key,
+		      const u8 raw_key[GHASH_BLOCK_SIZE]);
+
 /**
  * polyval_preparekey() - Prepare a POLYVAL key
  * @key: (output) The key structure to initialize
  * @raw_key: The raw hash key
  *

@@ -79,10 +117,22 @@ struct polyval_ctx {
  * Context: Any context.
  */
 void polyval_preparekey(struct polyval_key *key,
 			const u8 raw_key[POLYVAL_BLOCK_SIZE]);
 
+/**
+ * ghash_init() - Initialize a GHASH context for a new message
+ * @ctx: The context to initialize
+ * @key: The key to use.  Note that a pointer to the key is saved in the
+ *	 context, so the key must live at least as long as the context.
+ */
+static inline void ghash_init(struct ghash_ctx *ctx,
+			      const struct ghash_key *key)
+{
+	*ctx = (struct ghash_ctx){ .key = key };
+}
+
 /**
  * polyval_init() - Initialize a POLYVAL context for a new message
  * @ctx: The context to initialize
  * @key: The key to use.  Note that a pointer to the key is saved in the
  *	 context, so the key must live at least as long as the context.

@@ -123,10 +173,22 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx,
 					     struct polyval_elem *acc)
 {
 	*acc = ctx->acc;
 }
 
+/**
+ * ghash_update() - Update a GHASH context with message data
+ * @ctx: The context to update; must have been initialized
+ * @data: The message data
+ * @len: The data length in bytes.  Doesn't need to be block-aligned.
+ *
+ * This can be called any number of times.
+ *
+ * Context: Any context.
+ */
+void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len);
+
 /**
  * polyval_update() - Update a POLYVAL context with message data
  * @ctx: The context to update; must have been initialized
  * @data: The message data
  * @len: The data length in bytes.  Doesn't need to be block-aligned.

@@ -135,10 +197,24 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx,
  *
  * Context: Any context.
  */
 void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len);
 
+/**
+ * ghash_final() - Finish computing a GHASH value
+ * @ctx: The context to finalize
+ * @out: The output value
+ *
+ * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the
+ * final block is automatically zero-padded.
+ *
+ * After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+ *
+ * Context: Any context.
+ */
+void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]);
+
 /**
  * polyval_final() - Finish computing a POLYVAL value
  * @ctx: The context to finalize
  * @out: The output value
  *

@@ -149,10 +225,29 @@ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len);
  *
  * Context: Any context.
  */
 void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]);
 
+/**
+ * ghash() - Compute a GHASH value
+ * @key: The prepared key
+ * @data: The message data
+ * @len: The data length in bytes.  Doesn't need to be block-aligned.
+ * @out: The output value
+ *
+ * Context: Any context.
+ */
+static inline void ghash(const struct ghash_key *key, const u8 *data,
+			 size_t len, u8 out[GHASH_BLOCK_SIZE])
+{
+	struct ghash_ctx ctx;
+
+	ghash_init(&ctx, key);
+	ghash_update(&ctx, data, len);
+	ghash_final(&ctx, out);
+}
+
 /**
  * polyval() - Compute a POLYVAL value
  * @key: The prepared key
  * @data: The message data
  * @len: The data length in bytes.  Doesn't need to be block-aligned.

193 lines not on this page · raw message

186 lines not on this page · raw message

194 lines not on this page · raw message

index 454adcc62cc6..d7d787de7dd3 100644
@@ -34,11 +34,11 @@ MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
 
 #define RFC4106_NONCE_SIZE	4
 
 struct ghash_key {
 	be128	k;
-	u64	h[][2];
+	u64	h[1][2];
 };
 
 struct gcm_key {
 	u64	h[4][2];
 	u32	rk[AES_MAX_KEYLENGTH_U32];

@@ -49,16 +49,14 @@ struct gcm_key {
 struct arm_ghash_desc_ctx {
 	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
 };
 
 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
-				       u64 const h[][2], const char *head);
+				       u64 const h[4][2], const char *head);
 
 asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
-				      u64 const h[][2], const char *head);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
+				      u64 const h[1][2], const char *head);
 
 static int ghash_init(struct shash_desc *desc)
 {
 	struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc);
 

@@ -68,14 +66,11 @@ static int ghash_init(struct shash_desc *desc)
 
 static void ghash_do_update(int blocks, u64 dg[], const char *src,
 			    struct ghash_key *key, const char *head)
 {
 	kernel_neon_begin();
-	if (static_branch_likely(&use_p64))
-		pmull_ghash_update_p64(blocks, dg, src, key->h, head);
-	else
-		pmull_ghash_update_p8(blocks, dg, src, key->h, head);
+	pmull_ghash_update_p8(blocks, dg, src, key->h, head);
 	kernel_neon_end();
 }
 
 static int ghash_update(struct shash_desc *desc, const u8 *src,
 			unsigned int len)

@@ -145,23 +140,10 @@ static int ghash_setkey(struct crypto_shash *tfm,
 		return -EINVAL;
 
 	/* needed for the fallback */
 	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
 	ghash_reflect(key->h[0], &key->k);
-
-	if (static_branch_likely(&use_p64)) {
-		be128 h = key->k;
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[1], &h);
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[2], &h);
-
-		gf128mul_lle(&h, &key->k);
-		ghash_reflect(key->h[3], &h);
-	}
 	return 0;
 }
 
 static struct shash_alg ghash_alg = {
 	.digestsize		= GHASH_DIGEST_SIZE,

@@ -173,15 +155,15 @@ static struct shash_alg ghash_alg = {
 	.import			= ghash_import,
 	.descsize		= sizeof(struct arm_ghash_desc_ctx),
 	.statesize		= sizeof(struct ghash_desc_ctx),
 
 	.base.cra_name		= "ghash",
-	.base.cra_driver_name	= "ghash-ce",
+	.base.cra_driver_name	= "ghash-neon",
 	.base.cra_priority	= 300,
 	.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
-	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
+	.base.cra_ctxsize	= sizeof(struct ghash_key),
 	.base.cra_module	= THIS_MODULE,
 };
 
 void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src,
 		       struct gcm_key const *k, char *dst,

@@ -569,12 +551,10 @@ static int __init ghash_ce_mod_init(void)
 	if (elf_hwcap2 & HWCAP2_PMULL) {
 		err = crypto_register_aeads(gcm_aes_algs,
 					    ARRAY_SIZE(gcm_aes_algs));
 		if (err)
 			return err;
-		ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
-		static_branch_enable(&use_p64);
 	}
 
 	err = crypto_register_shash(&ghash_alg);
 	if (err)
 		goto err_aead;

213 lines not on this page · raw message

43 lines not on this page · raw message

241 lines not on this page · raw message

111 lines not on this page · raw message

index 42fb46bdc124..c74066d430fa 100644
@@ -28,38 +28,38 @@ MODULE_LICENSE("GPL v2");
 MODULE_ALIAS_CRYPTO("gcm(aes)");
 MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
 
 #define RFC4106_NONCE_SIZE	4
 
-struct ghash_key {
+struct arm_ghash_key {
 	be128			k;
-	u64			h[][2];
+	u64			h[4][2];
 };
 
 struct arm_ghash_desc_ctx {
 	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
 };
 
 struct gcm_aes_ctx {
 	struct aes_enckey	aes_key;
 	u8			nonce[RFC4106_NONCE_SIZE];
-	struct ghash_key	ghash_key;
+	struct arm_ghash_key	ghash_key;
 };
 
 asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
-				       u64 const h[][2], const char *head);
+				       u64 const h[4][2], const char *head);
 
 asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
-				  u64 const h[][2], u64 dg[], u8 ctr[],
+				  u64 const h[4][2], u64 dg[], u8 ctr[],
 				  u32 const rk[], int rounds, u8 tag[]);
 asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
-				 u64 const h[][2], u64 dg[], u8 ctr[],
+				 u64 const h[4][2], u64 dg[], u8 ctr[],
 				 u32 const rk[], int rounds, const u8 l[],
 				 const u8 tag[], u64 authsize);
 
 static void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
-				 struct ghash_key *key, const char *head)
+				 struct arm_ghash_key *key, const char *head)
 {
 	scoped_ksimd()
 		pmull_ghash_update_p64(blocks, dg, src, key->h, head);
 }
 

@@ -365,12 +365,11 @@ static struct aead_alg gcm_aes_algs[] = {{
 
 	.base.cra_name		= "gcm(aes)",
 	.base.cra_driver_name	= "gcm-aes-ce",
 	.base.cra_priority	= 300,
 	.base.cra_blocksize	= 1,
-	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx) +
-				  4 * sizeof(u64[2]),
+	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx),
 	.base.cra_module	= THIS_MODULE,
 }, {
 	.ivsize			= GCM_RFC4106_IV_SIZE,
 	.chunksize		= AES_BLOCK_SIZE,
 	.maxauthsize		= AES_BLOCK_SIZE,

@@ -381,12 +380,11 @@ static struct aead_alg gcm_aes_algs[] = {{
 
 	.base.cra_name		= "rfc4106(gcm(aes))",
 	.base.cra_driver_name	= "rfc4106-gcm-aes-ce",
 	.base.cra_priority	= 300,
 	.base.cra_blocksize	= 1,
-	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx) +
-				  4 * sizeof(u64[2]),
+	.base.cra_ctxsize	= sizeof(struct gcm_aes_ctx),
 	.base.cra_module	= THIS_MODULE,
 }};
 
 static int __init ghash_ce_mod_init(void)
 {

index 6862c605cc33..c68f5b6965fa 100644
@@ -1,8 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/types.h>
 #include <crypto/aes.h>
 
-extern struct shash_alg p8_ghash_alg;
 extern struct skcipher_alg p8_aes_cbc_alg;
 extern struct skcipher_alg p8_aes_ctr_alg;
 extern struct skcipher_alg p8_aes_xts_alg;

160 lines not on this page · raw message

index 7d2beb774f99..08da5311dfdf 100644
@@ -12,26 +12,21 @@
 #include <linux/types.h>
 #include <linux/err.h>
 #include <linux/cpufeature.h>
 #include <linux/crypto.h>
 #include <asm/cputable.h>
-#include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 
 #include "aesp8-ppc.h"
 
 static int __init p8_init(void)
 {
 	int ret;
 
-	ret = crypto_register_shash(&p8_ghash_alg);
-	if (ret)
-		goto err;
-
 	ret = crypto_register_skcipher(&p8_aes_cbc_alg);
 	if (ret)
-		goto err_unregister_ghash;
+		goto err;
 
 	ret = crypto_register_skcipher(&p8_aes_ctr_alg);
 	if (ret)
 		goto err_unregister_aes_cbc;
 

@@ -43,22 +38,19 @@ static int __init p8_init(void)
 
 err_unregister_aes_ctr:
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 err_unregister_aes_cbc:
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-err_unregister_ghash:
-	crypto_unregister_shash(&p8_ghash_alg);
 err:
 	return ret;
 }
 
 static void __exit p8_exit(void)
 {
 	crypto_unregister_skcipher(&p8_aes_xts_alg);
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-	crypto_unregister_shash(&p8_ghash_alg);
 }
 
 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init);
 module_exit(p8_exit);
 

index 5090fbaa87f8..650652dd6003 100644
@@ -39,10 +39,14 @@ struct polyval_elem {
  * struct ghash_key - Prepared key for GHASH
  *
  * Use ghash_preparekey() to initialize this.
  */
 struct ghash_key {
+#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
+	/** @htable: GHASH key format used by the POWER8 assembly code */
+	u64 htable[4][2];
+#endif
 	/** @h: The hash key H, in POLYVAL format */
 	struct polyval_elem h;
 };
 
 /**

109 lines not on this page · raw message

146 lines not on this page · raw message

index 650652dd6003..b798438cce23 100644
@@ -42,10 +42,13 @@ struct polyval_elem {
  */
 struct ghash_key {
 #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
 	/** @htable: GHASH key format used by the POWER8 assembly code */
 	u64 htable[4][2];
+#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
+	/** @h_raw: The hash key H, in GHASH format */
+	u8 h_raw[GHASH_BLOCK_SIZE];
 #endif
 	/** @h: The hash key H, in POLYVAL format */
 	struct polyval_elem h;
 };
 

57 lines not on this page · raw message

144 lines not on this page · raw message

index b798438cce23..0bc649d01e12 100644
@@ -42,11 +42,12 @@ struct polyval_elem {
  */
 struct ghash_key {
 #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
 	/** @htable: GHASH key format used by the POWER8 assembly code */
 	u64 htable[4][2];
-#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
+#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
+	(defined(CONFIG_RISCV) || defined(CONFIG_S390))
 	/** @h_raw: The hash key H, in GHASH format */
 	u8 h_raw[GHASH_BLOCK_SIZE];
 #endif
 	/** @h: The hash key H, in POLYVAL format */
 	struct polyval_elem h;

54 lines not on this page · raw message

163 lines not on this page · raw message

105 lines not on this page · raw message

670 lines not on this page · raw message

index fec950f1628b..0b0ad358e091 100644
@@ -4963,26 +4963,26 @@ static const struct alg_test_desc alg_test_descs[] = {
 			.kpp = __VECS(ffdhe8192_dh_tv_template)
 		}
 	}, {
 #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */
 		.alg = "gcm(aes)",
-		.generic_driver = "gcm_base(ctr(aes-lib),ghash-generic)",
+		.generic_driver = "gcm_base(ctr(aes-lib),ghash-lib)",
 		.test = alg_test_aead,
 		.fips_allowed = 1,
 		.suite = {
 			.aead = __VECS(aes_gcm_tv_template)
 		}
 	}, {
 		.alg = "gcm(aria)",
-		.generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)",
+		.generic_driver = "gcm_base(ctr(aria-generic),ghash-lib)",
 		.test = alg_test_aead,
 		.suite = {
 			.aead = __VECS(aria_gcm_tv_template)
 		}
 	}, {
 		.alg = "gcm(sm4)",
-		.generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)",
+		.generic_driver = "gcm_base(ctr(sm4-generic),ghash-lib)",
 		.test = alg_test_aead,
 		.suite = {
 			.aead = __VECS(sm4_gcm_tv_template)
 		}
 	}, {

@@ -5312,11 +5312,11 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.cipher = __VECS(sm4_ctr_rfc3686_tv_template)
 		}
 	}, {
 		.alg = "rfc4106(gcm(aes))",
-		.generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-generic))",
+		.generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-lib))",
 		.test = alg_test_aead,
 		.fips_allowed = 1,
 		.suite = {
 			.aead = {
 				____VECS(aes_gcm_rfc4106_tv_template),

@@ -5336,11 +5336,11 @@ static const struct alg_test_desc alg_test_descs[] = {
 				.aad_iv = 1,
 			}
 		}
 	}, {
 		.alg = "rfc4543(gcm(aes))",
-		.generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-generic))",
+		.generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-lib))",
 		.test = alg_test_aead,
 		.suite = {
 			.aead = {
 				____VECS(aes_gcm_rfc4543_tv_template),
 				.einval_allowed = 1,

index 2e2d97d17e6c..a0713aa21250 100644
@@ -1006,11 +1006,11 @@ static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm)
 	return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac-aes-lib)");
 }
 
 static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm)
 {
-	return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-generic)");
+	return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-lib)");
 }
 
 static struct skcipher_engine_alg skcipher_algs[] = {
 {
 	.base.init			= starfive_aes_ecb_init_tfm,

162 lines not on this page · raw message

index aded37546137..1773f5f71351 100644
@@ -1648,14 +1648,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
 
 	case 45:
 		ret = min(ret, tcrypt_test("rfc4309(ccm(aes))"));
 		break;
 
-	case 46:
-		ret = min(ret, tcrypt_test("ghash"));
-		break;
-
 	case 48:
 		ret = min(ret, tcrypt_test("sha3-224"));
 		break;
 
 	case 49:

@@ -2249,15 +2245,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
 		fallthrough;
 	case 317:
 		test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
 		if (mode > 300 && mode < 400) break;
 		fallthrough;
-	case 318:
-		klen = 16;
-		test_hash_speed("ghash", sec, generic_hash_speed_template);
-		if (mode > 300 && mode < 400) break;
-		fallthrough;
 	case 319:
 		test_hash_speed("crc32c", sec, generic_hash_speed_template);
 		if (mode > 300 && mode < 400) break;
 		fallthrough;
 	case 322:

index 0b0ad358e091..dd01f86dd6fe 100644
@@ -4983,16 +4983,10 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.generic_driver = "gcm_base(ctr(sm4-generic),ghash-lib)",
 		.test = alg_test_aead,
 		.suite = {
 			.aead = __VECS(sm4_gcm_tv_template)
 		}
-	}, {
-		.alg = "ghash",
-		.test = alg_test_hash,
-		.suite = {
-			.hash = __VECS(ghash_tv_template)
-		}
 	}, {
 		.alg = "hctr2(aes)",
 		.generic_driver = "hctr2_base(xctr(aes-lib),polyval-lib)",
 		.test = alg_test_skcipher,
 		.suite = {

119 lines not on this page · raw message

index b0853f7cada0..6ed2a8351902 100644
@@ -213,29 +213,18 @@ static inline void gf128mul_x_ble(le128 *r, const le128 *x)
 
 	r->a = cpu_to_le64((a << 1) | (b >> 63));
 	r->b = cpu_to_le64((b << 1) ^ _tt);
 }
 
-/* 4k table optimization */
-
-struct gf128mul_4k {
-	be128 t[256];
-};
-
-struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g);
-void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t);
 void gf128mul_x8_ble(le128 *r, const le128 *x);
-static inline void gf128mul_free_4k(struct gf128mul_4k *t)
-{
-	kfree_sensitive(t);
-}
-
 
 /* 64k table optimization, implemented for bbe */
 
 struct gf128mul_64k {
-	struct gf128mul_4k *t[16];
+	struct {
+		be128 t[256];
+	} *t[16];
 };
 
 /* First initialize with the constant factor with which you
  * want to multiply and then call gf128mul_64k_bbe with the other
  * factor in the first argument, and the table in the second.

92 lines not on this page · raw message

index 0bc649d01e12..41c557d55965 100644
@@ -6,15 +6,14 @@
  */
 
 #ifndef _CRYPTO_GF128HASH_H
 #define _CRYPTO_GF128HASH_H
 
+#include <crypto/ghash.h>
 #include <linux/string.h>
 #include <linux/types.h>
 
-#define GHASH_BLOCK_SIZE	16
-#define GHASH_DIGEST_SIZE	16
 #define POLYVAL_BLOCK_SIZE	16
 #define POLYVAL_DIGEST_SIZE	16
 
 /**
  * struct polyval_elem - An element of the POLYVAL finite field

index 043d938e9a2c..d187e5af9925 100644
@@ -4,21 +4,9 @@
  */
 
 #ifndef __CRYPTO_GHASH_H__
 #define __CRYPTO_GHASH_H__
 
-#include <linux/types.h>
-
 #define GHASH_BLOCK_SIZE	16
 #define GHASH_DIGEST_SIZE	16
 
-struct gf128mul_4k;
-
-struct ghash_ctx {
-	struct gf128mul_4k *gf128;
-};
-
-struct ghash_desc_ctx {
-	u8 buffer[GHASH_BLOCK_SIZE];
-};
-
 #endif

index e6c38d1d8a92..f522fff9231e 100644
@@ -23,10 +23,11 @@
 #include <linux/err.h>
 #include <crypto/algapi.h>
 #include <crypto/aes.h>
 #include <crypto/b128ops.h>
 #include <crypto/gcm.h>
+#include <crypto/gf128mul.h>
 #include <crypto/xts.h>
 #include <asm/cpu_device_id.h>
 #include <asm/simd.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/aead.h>

index b524e47bd4d0..1d5f39ff1dc4 100644
@@ -2,11 +2,11 @@
 #define _CRYPTO_GCM_H
 
 #include <linux/errno.h>
 
 #include <crypto/aes.h>
-#include <crypto/gf128mul.h>
+#include <crypto/gf128hash.h>
 
 #define GCM_AES_IV_SIZE 12
 #define GCM_RFC4106_IV_SIZE 8
 #define GCM_RFC4543_IV_SIZE 8
 

@@ -63,11 +63,11 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen)
 
 	return 0;
 }
 
 struct aesgcm_ctx {
-	be128			ghash_key;
+	struct ghash_key	ghash_key;
 	struct aes_enckey	aes_key;
 	unsigned int		authsize;
 };
 
 int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key,

102 lines not on this page · raw message

index c74066d430fa..eaf2932ceaf5 100644
@@ -35,10 +35,6 @@ struct arm_ghash_key {
 	u64			h[4][2];
 };
 
-struct arm_ghash_desc_ctx {
-	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
-};
-
 struct gcm_aes_ctx {
 	struct aes_enckey	aes_key;
 	u8			nonce[RFC4106_NONCE_SIZE];

index cb929bed29d5..c33c8cbe51fe 100644
@@ -12,7 +12,7 @@
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 
 void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
-			   const u8 *src, const struct polyval_elem *k);
+			   const u8 *src, const struct polyval_elem *h);
 
 #define ghash_blocks_arch ghash_blocks_arch
 static void ghash_blocks_arch(struct polyval_elem *acc,

index d5ef1b1b77e1..b2c85585b758 100644
@@ -12,14 +12,14 @@
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
 
+asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
+				      const u8 *src,
+				      const struct polyval_elem *h);
 asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
 				  const struct polyval_elem *b);
 asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
 				     const struct polyval_key *key,
 				     const u8 *data, size_t nblocks);
-asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
-				      const u8 *src,
-				      const struct polyval_elem *k);
 
 #define polyval_preparekey_arch polyval_preparekey_arch
 static void polyval_preparekey_arch(struct polyval_key *key,

@@ -91,8 +91,8 @@ static void ghash_blocks_arch(struct polyval_elem *acc,
 	if (static_branch_likely(&have_asimd) && may_use_simd()) {
 		do {
 			/* Allow rescheduling every 4 KiB. */
-			size_t n =
-				min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE);
+			size_t n = min_t(size_t, nblocks,
+					 4096 / GHASH_BLOCK_SIZE);
 
 			scoped_ksimd()
 				pmull_ghash_update_p8(n, acc, data, &key->h);