crypto: lib - Add partial block helper

PATCH Superseded Herbert Xu 2 participants 13 messages Apr 27, 2025

Subsystems (8)

Arm (32-bit) x86 PowerPC MIPS +4 more subsystems

lore

Share & tools

lore

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #0 lore raw

v2 cover crypto: lib - Add partial block helper

v2:
- Remove the polyval patches.
- Rename one-block key to raw_key.
- Rename poly1305_block to poly1305_blocks.
- Fix the arch/generic if clause in poly1305_blocks.
- Rewrite crypto/chacha20poly1305 to use lib/crypto poly1305.
- Remove shash poly1305.

This is based on

	https://patchwork.kernel.org/project/linux-crypto/list/?series=955753
	https://patchwork.kernel.org/project/linux-crypto/list/?series=957401

This series introduces a partial block helper for lib/crypto hash algorithms based on the one from sha256_base.

It then uses it on poly1305 to eliminate duplication between architectures. In particular, instead of having complete update functions for each architecture, reduce it to a block function per architecture instead. The partial block handling is handled by the generic library layer.

The poly1305 implementation was anomalous due to the inability to call setkey in softirq. It also has just a single user, which is chacha20poly1305 that is hard-coded to use poly1305. Replace the gratuitous use of ahash in chacha20poly1305 with the lib/crypto poly1305 instead.

This then allows the shash poly1305 to be removed.

Herbert Xu (11):
crypto: lib/sha256 - Move partial block handling out
crypto: lib/poly1305 - Add block-only interface
crypto: arm/poly1305 - Add block-only interface
crypto: arm64/poly1305 - Add block-only interface
crypto: mips/poly1305 - Add block-only interface
crypto: powerpc/poly1305 - Add block-only interface
crypto: x86/poly1305 - Add block-only interface
crypto: chacha20poly1305 - Use lib/crypto poly1305
crypto: testmgr - Remove poly1305
crypto: poly1305 - Remove algorithm
crypto: lib/poly1305 - Use block-only interface

 arch/arm/lib/crypto/poly1305-armv4.pl         |   4 +-
 arch/arm/lib/crypto/poly1305-glue.c           | 113 ++----
 arch/arm64/lib/crypto/Makefile                |   3 +-
 arch/arm64/lib/crypto/poly1305-glue.c         | 105 ++----
 arch/mips/lib/crypto/poly1305-glue.c          |  75 +---
 arch/mips/lib/crypto/poly1305-mips.pl         |  12 +-
 arch/powerpc/lib/crypto/poly1305-p10-glue.c   | 109 ++----
 .../lib/crypto/poly1305-x86_64-cryptogams.pl  |  33 +-
 arch/x86/lib/crypto/poly1305_glue.c           | 169 +++------
 crypto/Kconfig                                |  14 +-
 crypto/Makefile                               |   2 -
 crypto/chacha20poly1305.c                     | 323 ++++--------------
 crypto/poly1305.c                             | 152 ---------
 crypto/testmgr.c                              |   6 -
 crypto/testmgr.h                              | 288 ----------------
 include/crypto/internal/blockhash.h           |  52 +++
 include/crypto/internal/poly1305.h            |  28 +-
 include/crypto/poly1305.h                     |  60 +---
 include/crypto/sha2.h                         |   9 +-
 include/crypto/sha256_base.h                  |  38 +--
 lib/crypto/poly1305.c                         |  83 ++---
 21 files changed, 396 insertions(+), 1282 deletions(-)
 delete mode 100644 crypto/poly1305.c
 create mode 100644 include/crypto/internal/blockhash.h

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #1 lore raw

PATCH v2 1/11 crypto: lib/sha256 - Move partial block handling out

Extract the common partial block handling into a helper macro that can be reused by other library code.

Also delete the unused sha256_base_do_finalize function.

Signed-off-by Herbert Xu <[email protected]>

---
 include/crypto/internal/blockhash.h | 52 +++++++++++++++++++++++++++++
 include/crypto/sha2.h               |  9 +++--
 include/crypto/sha256_base.h        | 38 ++-------------------
 3 files changed, 62 insertions(+), 37 deletions(-)
 create mode 100644 include/crypto/internal/blockhash.h

include/crypto/internal/blockhash.h new file +52 -0

new file mode 100644
index 000000000000..4184e2337d68
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Handle partial blocks for block hash.
+ *
+ * Copyright (c) 2015 Linaro Ltd <[email protected]>
+ * Copyright (c) 2025 Herbert Xu <[email protected]>
+ */
+
+#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H
+#define _CRYPTO_INTERNAL_BLOCKHASH_H
+
+#include <linux/string.h>
+#include <linux/types.h>
+
+#define BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, dv, buf,	\
+			       buflen)					\
+	({								\
+		unsigned int _nbytes = (nbytes);			\
+		unsigned int _buflen = (buflen);			\
+		typeof(block) _block = (block);				\
+		typeof(state) _state = (state); 			\
+		unsigned int _bs = (bs);				\
+		unsigned int _dv = (dv);				\
+		const u8 *_src = (src);					\
+		u8 *_buf = (buf);					\
+		while ((_buflen + _nbytes) >= _bs) {			\
+			unsigned int len = _nbytes;			\
+			const u8 *data = _src;				\
+			int blocks, remain;				\
+			if (_buflen) {					\
+				remain = _bs - _buflen;			\
+				memcpy(_buf + _buflen, _src, remain);	\
+				data = _buf;				\
+				len = _bs;				\
+			}						\
+			remain = len % bs;				\
+			blocks = (len - remain) / _dv;			\
+			_block(_state, data, blocks);			\
+			_src += len - remain - _buflen;			\
+			_nbytes -= len - remain - _buflen;		\
+			_buflen = 0;					\
+		}							\
+		memcpy(_buf + _buflen, _src, _nbytes);			\
+		_buflen += _nbytes;					\
+	})
+
+#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen)
+#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen)
+
+#endif	/* _CRYPTO_INTERNAL_BLOCKHASH_H */

include/crypto/sha2.h +7 -2

index abbd882f7849..f873c2207b1e 100644
@@ -71,8 +71,13 @@ struct crypto_sha256_state {
 };
 
 struct sha256_state {
-	u32 state[SHA256_DIGEST_SIZE / 4];
-	u64 count;
+	union {
+		struct crypto_sha256_state ctx;
+		struct {
+			u32 state[SHA256_DIGEST_SIZE / 4];
+			u64 count;
+		};
+	};
 	u8 buf[SHA256_BLOCK_SIZE];
 };

include/crypto/sha256_base.h +3 ~~-35~~

index 08cd5e41d4fd..9f284bed5a51 100644
@@ -8,6 +8,7 @@
 #ifndef _CRYPTO_SHA256_BASE_H
 #define _CRYPTO_SHA256_BASE_H
 
+#include <crypto/internal/blockhash.h>
 #include <crypto/internal/hash.h>
 #include <crypto/sha2.h>
 #include <linux/math.h>

@@ -40,35 +41,10 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx,
 					    sha256_block_fn *block_fn)
 {
 	unsigned int partial = sctx->count % SHA256_BLOCK_SIZE;
-	struct crypto_sha256_state *state = (void *)sctx;
 
 	sctx->count += len;
-
-	if (unlikely((partial + len) >= SHA256_BLOCK_SIZE)) {
-		int blocks;
-
-		if (partial) {
-			int p = SHA256_BLOCK_SIZE - partial;
-
-			memcpy(sctx->buf + partial, data, p);
-			data += p;
-			len -= p;
-
-			block_fn(state, sctx->buf, 1);
-		}
-
-		blocks = len / SHA256_BLOCK_SIZE;
-		len %= SHA256_BLOCK_SIZE;
-
-		if (blocks) {
-			block_fn(state, data, blocks);
-			data += blocks * SHA256_BLOCK_SIZE;
-		}
-		partial = 0;
-	}
-	if (len)
-		memcpy(sctx->buf + partial, data, len);
-
+	BLOCK_HASH_UPDATE_BLOCKS(block_fn, &sctx->ctx, data, len,
+				 SHA256_BLOCK_SIZE, sctx->buf, partial);
 	return 0;
 }

@@ -140,14 +116,6 @@ static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx,
 	return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn);
 }
 
-static inline int sha256_base_do_finalize(struct shash_desc *desc,
-					  sha256_block_fn *block_fn)
-{
-	struct sha256_state *sctx = shash_desc_ctx(desc);
-
-	return lib_sha256_base_do_finalize(sctx, block_fn);
-}
-
 static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4],
 				       u8 *out, unsigned int digest_size)
 {

Signature · 1 line

2.39.5

Eric Biggers

maintainer

2025-04-27 01:24 UTC #2 lore raw

Re: [v2 PATCH 01/11] crypto: lib/sha256 - Move partial block handling out

Quoted 59 lines · Herbert Xu

Extract the common partial block handling into a helper macro that can be reused by other library code.

Also delete the unused sha256_base_do_finalize function.

Signed-off-by Herbert Xu <[email protected]>

---
 include/crypto/internal/blockhash.h | 52 +++++++++++++++++++++++++++++
 include/crypto/sha2.h               |  9 +++--
 include/crypto/sha256_base.h        | 38 ++-------------------
 3 files changed, 62 insertions(+), 37 deletions(-)
 create mode 100644 include/crypto/internal/blockhash.h

include/crypto/internal/blockhash.h new file +40 -0

new file mode 100644
index 000000000000..4184e2337d68
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Handle partial blocks for block hash.
+ *
+ * Copyright (c) 2015 Linaro Ltd <[email protected]>
+ * Copyright (c) 2025 Herbert Xu <[email protected]>
+ */
+
+#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H
+#define _CRYPTO_INTERNAL_BLOCKHASH_H
+
+#include <linux/string.h>
+#include <linux/types.h>
+
+#define BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, dv, buf,	\
+			       buflen)					\
+	({								\
+		unsigned int _nbytes = (nbytes);			\
+		unsigned int _buflen = (buflen);			\
+		typeof(block) _block = (block);				\
+		typeof(state) _state = (state); 			\
+		unsigned int _bs = (bs);				\
+		unsigned int _dv = (dv);				\
+		const u8 *_src = (src);					\
+		u8 *_buf = (buf);					\
+		while ((_buflen + _nbytes) >= _bs) {			\
+			unsigned int len = _nbytes;			\
+			const u8 *data = _src;				\
+			int blocks, remain;				\
+			if (_buflen) {					\
+				remain = _bs - _buflen;			\
+				memcpy(_buf + _buflen, _src, remain);	\
+				data = _buf;				\
+				len = _bs;				\
+			}						\
+			remain = len % bs;				\
+			blocks = (len - remain) / _dv;			\
+			_block(_state, data, blocks);			\
+			_src += len - remain - _buflen;			\
+			_nbytes -= len - remain - _buflen;		\

include/crypto/internal/blockhash.h line 50

Quoted 10 lines

include/crypto/internal/blockhash.h new file +10 -0

+			_buflen = 0;					\
+		}							\
+		memcpy(_buf + _buflen, _src, _nbytes);			\
+		_buflen += _nbytes;					\
+	})
+
+#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen)
+#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen)

Again, these pointless macros just obfuscate things. And there's no reason to still be futzing around with SHA-256 when my patchset reworks it anyway.

- Eric

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #3 lore raw

PATCH v2 2/11 crypto: lib/poly1305 - Add block-only interface

In reply to Herbert Xu

Add a block-only interface for poly1305. Implement the generic code first.

Also use the generic partial block helper.

Signed-off-by Herbert Xu <[email protected]>

---
 include/crypto/internal/poly1305.h | 28 ++++++++++++++--
 include/crypto/poly1305.h          | 25 ++++++++++----
 lib/crypto/poly1305.c              | 54 +++++++++++++-----------------
 3 files changed, 68 insertions(+), 39 deletions(-)

include/crypto/internal/poly1305.h +26 -2

index e614594f88c1..c60315f47562 100644
@@ -6,9 +6,8 @@
 #ifndef _CRYPTO_INTERNAL_POLY1305_H
 #define _CRYPTO_INTERNAL_POLY1305_H
 
-#include <linux/unaligned.h>
-#include <linux/types.h>
 #include <crypto/poly1305.h>
+#include <linux/types.h>
 
 /*
  * Poly1305 core functions.  These only accept whole blocks; the caller must

@@ -31,4 +30,29 @@ void poly1305_core_blocks(struct poly1305_state *state,
 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
 			void *dst);
 
+void poly1305_block_init_arch(struct poly1305_block_state *state,
+			      const u8 raw_key[POLY1305_BLOCK_SIZE]);
+void poly1305_block_init_generic(struct poly1305_block_state *state,
+				 const u8 raw_key[POLY1305_BLOCK_SIZE]);
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+			  unsigned int len, u32 padbit);
+
+static inline void poly1305_blocks_generic(struct poly1305_block_state *state,
+					   const u8 *src, unsigned int len,
+					   u32 padbit)
+{
+	poly1305_core_blocks(&state->h, &state->core_r, src,
+			     len / POLY1305_BLOCK_SIZE, padbit);
+}
+
+void poly1305_emit_arch(const struct poly1305_state *state,
+			u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]);
+
+static inline void poly1305_emit_generic(const struct poly1305_state *state,
+					 u8 digest[POLY1305_DIGEST_SIZE],
+					 const u32 nonce[4])
+{
+	poly1305_core_emit(state, nonce, digest);
+}
+
 #endif

include/crypto/poly1305.h +19 -6

index 6e21ec2d1dc2..027d74842cd5 100644
@@ -7,7 +7,6 @@
 #define _CRYPTO_POLY1305_H
 
 #include <linux/types.h>
-#include <linux/crypto.h>
 
 #define POLY1305_BLOCK_SIZE	16
 #define POLY1305_KEY_SIZE	32

@@ -38,6 +37,17 @@ struct poly1305_state {
 	};
 };
 
+/* Combined state for block function. */
+struct poly1305_block_state {
+	/* accumulator */
+	struct poly1305_state h;
+	/* key */
+	union {
+		struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
+		struct poly1305_core_key core_r;
+	};
+};
+
 struct poly1305_desc_ctx {
 	/* partial buffer */
 	u8 buf[POLY1305_BLOCK_SIZE];

@@ -45,12 +55,15 @@ struct poly1305_desc_ctx {
 	unsigned int buflen;
 	/* finalize key */
 	u32 s[4];
-	/* accumulator */
-	struct poly1305_state h;
-	/* key */
 	union {
-		struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
-		struct poly1305_core_key core_r;
+		struct {
+			struct poly1305_state h;
+			union {
+				struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
+				struct poly1305_core_key core_r;
+			};
+		};
+		struct poly1305_block_state state;
 	};
 };

lib/crypto/poly1305.c +23 ~~-31~~

87 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #4 lore raw

PATCH v2 3/11 crypto: arm/poly1305 - Add block-only interface

In reply to Herbert Xu

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by Herbert Xu <[email protected]>

---
 arch/arm/lib/crypto/poly1305-armv4.pl |  4 +-
 arch/arm/lib/crypto/poly1305-glue.c   | 78 +++++++++++++++------------
 2 files changed, 47 insertions(+), 35 deletions(-)

arch/arm/lib/crypto/poly1305-armv4.pl +2 -2

index 6d79498d3115..d57c6e2fc84a 100644
@@ -43,9 +43,9 @@ $code.=<<___;
 #else
 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
 # define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__
-# define poly1305_init   poly1305_init_arm
+# define poly1305_init   poly1305_block_init_arch
 # define poly1305_blocks poly1305_blocks_arm
-# define poly1305_emit   poly1305_emit_arm
+# define poly1305_emit   poly1305_emit_arch
 .globl	poly1305_blocks_neon
 #endif

arch/arm/lib/crypto/poly1305-glue.c +45 ~~-33~~

126 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #5 lore raw

PATCH v2 4/11 crypto: arm64/poly1305 - Add block-only interface

In reply to Herbert Xu

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by Herbert Xu <[email protected]>

---
 arch/arm64/lib/crypto/Makefile        |  3 +-
 arch/arm64/lib/crypto/poly1305-glue.c | 71 ++++++++++++++++-----------
 2 files changed, 45 insertions(+), 29 deletions(-)

arch/arm64/lib/crypto/Makefile +2 -1

index ac624c3effda..6207088397a7 100644
@@ -5,7 +5,8 @@ chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
 
 obj-$(CONFIG_CRYPTO_POLY1305_NEON) += poly1305-neon.o
 poly1305-neon-y := poly1305-core.o poly1305-glue.o
-AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_init_arm64
+AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch
+AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch
 
 quiet_cmd_perlasm = PERLASM $@
       cmd_perlasm = $(PERL) $(<) void $(@)

arch/arm64/lib/crypto/poly1305-glue.c +43 ~~-28~~

114 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #6 lore raw

PATCH v2 5/11 crypto: mips/poly1305 - Add block-only interface

In reply to Herbert Xu

Add block-only interface.

Signed-off-by Herbert Xu <[email protected]>

---
 arch/mips/lib/crypto/poly1305-glue.c  | 29 ++++++++++++++++++---------
 arch/mips/lib/crypto/poly1305-mips.pl | 12 +++++------
 2 files changed, 26 insertions(+), 15 deletions(-)

arch/mips/lib/crypto/poly1305-glue.c +20 -9

index 576e7a58e0b1..2fea4cacfe27 100644
@@ -5,23 +5,33 @@
  * Copyright (C) 2019 Linaro Ltd. <[email protected]>
  */
 
-#include <crypto/poly1305.h>
+#include <crypto/internal/poly1305.h>
 #include <linux/cpufeature.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
 
-asmlinkage void poly1305_init_mips(void *state, const u8 *key);
-asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit);
-asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce);
+asmlinkage void poly1305_block_init_arch(
+	struct poly1305_block_state *state,
+	const u8 raw_key[POLY1305_BLOCK_SIZE]);
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+asmlinkage void poly1305_blocks_arch(struct poly1305_block_state *state,
+				     const u8 *src, u32 len, u32 hibit);
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+				   u8 digest[POLY1305_DIGEST_SIZE],
+				   const u32 nonce[4]);
+EXPORT_SYMBOL_GPL(poly1305_emit_arch);
 
 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
 {
-	poly1305_init_mips(&dctx->h, key);
 	dctx->s[0] = get_unaligned_le32(key + 16);
 	dctx->s[1] = get_unaligned_le32(key + 20);
 	dctx->s[2] = get_unaligned_le32(key + 24);
 	dctx->s[3] = get_unaligned_le32(key + 28);
 	dctx->buflen = 0;
+	poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);

@@ -37,7 +47,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
 		dctx->buflen += bytes;
 
 		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			poly1305_blocks_mips(&dctx->h, dctx->buf,
+			poly1305_blocks_arch(&dctx->state, dctx->buf,
 					     POLY1305_BLOCK_SIZE, 1);
 			dctx->buflen = 0;
 		}

@@ -46,7 +56,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
 	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
 		unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
 
-		poly1305_blocks_mips(&dctx->h, src, len, 1);
+		poly1305_blocks_arch(&dctx->state, src, len, 1);
 		src += len;
 		nbytes %= POLY1305_BLOCK_SIZE;
 	}

@@ -64,10 +74,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_emit_mips(&dctx->h, dst, dctx->s);
+	poly1305_emit_arch(&dctx->h, dst, dctx->s);
 	*dctx = (struct poly1305_desc_ctx){};
 }
 EXPORT_SYMBOL(poly1305_final_arch);

arch/mips/lib/crypto/poly1305-mips.pl +6 -6

index b05bab884ed2..399f10c3e385 100644
@@ -93,9 +93,9 @@ $code.=<<___;
 #endif
 
 #ifdef	__KERNEL__
-# define poly1305_init   poly1305_init_mips
-# define poly1305_blocks poly1305_blocks_mips
-# define poly1305_emit   poly1305_emit_mips
+# define poly1305_init   poly1305_block_init_arch
+# define poly1305_blocks poly1305_blocks_arch
+# define poly1305_emit   poly1305_emit_arch
 #endif
 
 #if defined(__MIPSEB__) && !defined(MIPSEB)

@@ -565,9 +565,9 @@ $code.=<<___;
 #endif
 
 #ifdef	__KERNEL__
-# define poly1305_init   poly1305_init_mips
-# define poly1305_blocks poly1305_blocks_mips
-# define poly1305_emit   poly1305_emit_mips
+# define poly1305_init   poly1305_block_init_arch
+# define poly1305_blocks poly1305_blocks_arch
+# define poly1305_emit   poly1305_emit_arch
 #endif
 
 #if defined(__MIPSEB__) && !defined(MIPSEB)

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #7 lore raw

PATCH v2 6/11 crypto: powerpc/poly1305 - Add block-only interface

In reply to Herbert Xu

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by Herbert Xu <[email protected]>

---
 arch/powerpc/lib/crypto/poly1305-p10-glue.c | 84 ++++++++++++---------
 1 file changed, 49 insertions(+), 35 deletions(-)

arch/powerpc/lib/crypto/poly1305-p10-glue.c +49 ~~-35~~

index 00617f4c58e6..708435beaba6 100644
@@ -4,19 +4,20 @@
  *
  * Copyright 2023- IBM Corp. All rights reserved.
  */
+#include <asm/switch_to.h>
+#include <crypto/internal/poly1305.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/jump_label.h>
-#include <crypto/internal/simd.h>
-#include <crypto/poly1305.h>
-#include <linux/cpufeature.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
-#include <asm/simd.h>
-#include <asm/switch_to.h>
 
-asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
-asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
-asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
+asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
+asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+				   u8 digest[POLY1305_DIGEST_SIZE],
+				   const u32 nonce[4]);
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);

@@ -32,22 +33,49 @@ static void vsx_end(void)
 	preempt_enable();
 }
 
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+void poly1305_block_init_arch(struct poly1305_block_state *dctx,
+			      const u8 raw_key[POLY1305_BLOCK_SIZE])
 {
 	if (!static_key_enabled(&have_p10))
-		return poly1305_init_generic(dctx, key);
+		return poly1305_block_init_generic(dctx, raw_key);
 
 	dctx->h = (struct poly1305_state){};
-	dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0);
-	dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8);
+	dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
+	dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
+}
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+
+void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+{
 	dctx->s[0] = get_unaligned_le32(key + 16);
 	dctx->s[1] = get_unaligned_le32(key + 20);
 	dctx->s[2] = get_unaligned_le32(key + 24);
 	dctx->s[3] = get_unaligned_le32(key + 28);
 	dctx->buflen = 0;
+	poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+			  unsigned int len, u32 padbit)
+{
+	if (!static_key_enabled(&have_p10))
+		return poly1305_blocks_generic(state, src, len, padbit);
+	vsx_begin();
+	if (len >= POLY1305_BLOCK_SIZE * 4) {
+		poly1305_p10le_4blocks(state, src, len);
+		src += len - (len % (POLY1305_BLOCK_SIZE * 4));
+		len %= POLY1305_BLOCK_SIZE * 4;
+	}
+	while (len >= POLY1305_BLOCK_SIZE) {
+		poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit);
+		len -= POLY1305_BLOCK_SIZE;
+		src += POLY1305_BLOCK_SIZE;
+	}
+	vsx_end();
+}
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+
 void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
 			  const u8 *src, unsigned int srclen)
 {

@@ -64,28 +92,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
 		dctx->buflen += bytes;
 		if (dctx->buflen < POLY1305_BLOCK_SIZE)
 			return;
-		vsx_begin();
-		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
-		vsx_end();
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 1);
 		dctx->buflen = 0;
 	}
 
 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
-		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-		if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
-			vsx_begin();
-			poly1305_p10le_4blocks(&dctx->h, src, srclen);
-			vsx_end();
-			src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
-			srclen %= POLY1305_BLOCK_SIZE * 4;
-		}
-		while (srclen >= POLY1305_BLOCK_SIZE) {
-			vsx_begin();
-			poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1);
-			vsx_end();
-			srclen -= POLY1305_BLOCK_SIZE;
-			src += POLY1305_BLOCK_SIZE;
-		}
+		poly1305_blocks_arch(&dctx->state, src, srclen, 1);
+		src += srclen - (srclen % POLY1305_BLOCK_SIZE);
+		srclen %= POLY1305_BLOCK_SIZE;
 	}
 
 	if (unlikely(srclen)) {

@@ -104,12 +119,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		vsx_begin();
-		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-		vsx_end();
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_emit_64(&dctx->h, &dctx->s, dst);
+	poly1305_emit_arch(&dctx->h, dst, dctx->s);
 }
 EXPORT_SYMBOL(poly1305_final_arch);

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #8 lore raw

PATCH v2 7/11 crypto: x86/poly1305 - Add block-only interface

In reply to Herbert Xu

Add block-only interface.

Also remove the unnecessary SIMD fallback path.

Signed-off-by Herbert Xu <[email protected]>

---
 .../lib/crypto/poly1305-x86_64-cryptogams.pl  |  33 +++--
 arch/x86/lib/crypto/poly1305_glue.c           | 125 +++++++-----------
 2 files changed, 71 insertions(+), 87 deletions(-)

arch/x86/lib/crypto/poly1305-x86_64-cryptogams.pl +23 ~~-10~~

index 409ec6955733..501827254fed 100644
@@ -118,6 +118,19 @@ sub declare_function() {
 	}
 }
 
+sub declare_typed_function() {
+	my ($name, $align, $nargs) = @_;
+	if($kernel) {
+		$code .= "SYM_TYPED_FUNC_START($name)\n";
+		$code .= ".L$name:\n";
+	} else {
+		$code .= ".globl	$name\n";
+		$code .= ".type	$name,\@function,$nargs\n";
+		$code .= ".align	$align\n";
+		$code .= "$name:\n";
+	}
+}
+
 sub end_function() {
 	my ($name) = @_;
 	if($kernel) {

@@ -128,7 +141,7 @@ sub end_function() {
 }
 
 $code.=<<___ if $kernel;
-#include <linux/linkage.h>
+#include <linux/cfi_types.h>
 ___
 
 if ($avx) {

@@ -236,14 +249,14 @@ ___
 $code.=<<___ if (!$kernel);
 .extern	OPENSSL_ia32cap_P
 
-.globl	poly1305_init_x86_64
-.hidden	poly1305_init_x86_64
+.globl	poly1305_block_init_arch
+.hidden	poly1305_block_init_arch
 .globl	poly1305_blocks_x86_64
 .hidden	poly1305_blocks_x86_64
 .globl	poly1305_emit_x86_64
 .hidden	poly1305_emit_x86_64
 ___
-&declare_function("poly1305_init_x86_64", 32, 3);
+&declare_typed_function("poly1305_block_init_arch", 32, 3);
 $code.=<<___;
 	xor	%eax,%eax
 	mov	%rax,0($ctx)		# initialize hash value

@@ -298,7 +311,7 @@ $code.=<<___;
 .Lno_key:
 	RET
 ___
-&end_function("poly1305_init_x86_64");
+&end_function("poly1305_block_init_arch");
 
 &declare_function("poly1305_blocks_x86_64", 32, 4);
 $code.=<<___;

@@ -4105,9 +4118,9 @@ avx_handler:
 
 .section	.pdata
 .align	4
-	.rva	.LSEH_begin_poly1305_init_x86_64
-	.rva	.LSEH_end_poly1305_init_x86_64
-	.rva	.LSEH_info_poly1305_init_x86_64
+	.rva	.LSEH_begin_poly1305_block_init_arch
+	.rva	.LSEH_end_poly1305_block_init_arch
+	.rva	.LSEH_info_poly1305_block_init_arch
 
 	.rva	.LSEH_begin_poly1305_blocks_x86_64
 	.rva	.LSEH_end_poly1305_blocks_x86_64

@@ -4155,10 +4168,10 @@ ___
 $code.=<<___;
 .section	.xdata
 .align	8
-.LSEH_info_poly1305_init_x86_64:
+.LSEH_info_poly1305_block_init_arch:
 	.byte	9,0,0,0
 	.rva	se_handler
-	.rva	.LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64
+	.rva	.LSEH_begin_poly1305_block_init_arch,.LSEH_begin_poly1305_block_init_arch
 
 .LSEH_info_poly1305_blocks_x86_64:
 	.byte	9,0,0,0

arch/x86/lib/crypto/poly1305_glue.c +48 ~~-77~~

194 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #9 lore raw

PATCH v2 8/11 crypto: chacha20poly1305 - Use lib/crypto poly1305

In reply to Herbert Xu

Since the poly1305 algorithm is fixed, there is no point in going through the Crypto API for it. Use the lib/crypto poly1305 interface instead.

For compatiblity keep the poly1305 parameter in the algorithm name.

Signed-off-by Herbert Xu <[email protected]>

---
 crypto/Kconfig            |   2 +-
 crypto/chacha20poly1305.c | 323 ++++++++------------------------------
 2 files changed, 67 insertions(+), 258 deletions(-)

crypto/Kconfig +1 -1

index 9878286d1d68..f87e2a26d2dd 100644
@@ -784,8 +784,8 @@ config CRYPTO_AEGIS128_SIMD
 config CRYPTO_CHACHA20POLY1305
 	tristate "ChaCha20-Poly1305"
 	select CRYPTO_CHACHA20
-	select CRYPTO_POLY1305
 	select CRYPTO_AEAD
+	select CRYPTO_LIB_POLY1305
 	select CRYPTO_MANAGER
 	help
 	  ChaCha20 stream cipher and Poly1305 authenticator combined

crypto/chacha20poly1305.c +66 ~~-257~~

453 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #10 lore raw

PATCH v2 9/11 crypto: testmgr - Remove poly1305

In reply to Herbert Xu

As poly1305 no longer has any in-kernel users, remove its tests.

Signed-off-by Herbert Xu <[email protected]>

---
 crypto/testmgr.c |   6 -
 crypto/testmgr.h | 288 -----------------------------------------------
 2 files changed, 294 deletions(-)

crypto/testmgr.c +0 -6

index 82977ea25db3..f100be516f52 100644
@@ -5406,12 +5406,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "pkcs1pad(rsa)",
 		.test = alg_test_null,
 		.fips_allowed = 1,
-	}, {
-		.alg = "poly1305",
-		.test = alg_test_hash,
-		.suite = {
-			.hash = __VECS(poly1305_tv_template)
-		}
 	}, {
 		.alg = "polyval",
 		.test = alg_test_hash,

crypto/testmgr.h +0 ~~-288~~

294 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #11 lore raw

PATCH v2 10/11 crypto: poly1305 - Remove algorithm

In reply to Herbert Xu

As there are no in-kernel users of the Crypto API poly1305 left, remove it.

Signed-off-by Herbert Xu <[email protected]>

---
 crypto/Kconfig    |  12 ----
 crypto/Makefile   |   2 -
 crypto/poly1305.c | 152 ----------------------------------------------
 3 files changed, 166 deletions(-)
 delete mode 100644 crypto/poly1305.c

crypto/Kconfig +0 ~~-12~~

index f87e2a26d2dd..3cb5563dc4ab 100644
@@ -953,18 +953,6 @@ config CRYPTO_POLYVAL
 	  This is used in HCTR2.  It is not a general-purpose
 	  cryptographic hash function.
 
-config CRYPTO_POLY1305
-	tristate "Poly1305"
-	select CRYPTO_HASH
-	select CRYPTO_LIB_POLY1305
-	select CRYPTO_LIB_POLY1305_GENERIC
-	help
-	  Poly1305 authenticator algorithm (RFC7539)
-
-	  Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein.
-	  It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use
-	  in IETF protocols. This is the portable C implementation of Poly1305.
-
 config CRYPTO_RMD160
 	tristate "RIPEMD-160"
 	select CRYPTO_HASH

crypto/Makefile +0 -2

index 5d2f2a28d8a0..587bc74b6d74 100644
@@ -149,8 +149,6 @@ obj-$(CONFIG_CRYPTO_SEED) += seed.o
 obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o
 obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o
 CFLAGS_chacha.o += -DARCH=$(ARCH)
-obj-$(CONFIG_CRYPTO_POLY1305) += poly1305.o
-CFLAGS_poly1305.o += -DARCH=$(ARCH)
 obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o
 obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
 obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o

crypto/poly1305.c deleted +0 ~~-152~~

152 lines not on this page · raw message

Signature · 1 line

2.39.5

Herbert Xu

author maintainer

2025-04-27 01:00 UTC #12 lore raw

PATCH v2 11/11 crypto: lib/poly1305 - Use block-only interface

In reply to Herbert Xu

Now that every architecture provides a block function, use that to implement the lib/poly1305 and remove the old per-arch code.

Signed-off-by Herbert Xu <[email protected]>

---
 arch/arm/lib/crypto/poly1305-glue.c         | 57 -------------------
 arch/arm64/lib/crypto/poly1305-glue.c       | 58 -------------------
 arch/mips/lib/crypto/poly1305-glue.c        | 60 --------------------
 arch/powerpc/lib/crypto/poly1305-p10-glue.c | 63 ---------------------
 arch/x86/lib/crypto/poly1305_glue.c         | 60 --------------------
 include/crypto/poly1305.h                   | 53 ++---------------
 lib/crypto/poly1305.c                       | 39 ++++++++-----
 7 files changed, 32 insertions(+), 358 deletions(-)

arch/arm/lib/crypto/poly1305-glue.c +0 ~~-57~~

index 3ee16048ec7c..91da42b26d9c 100644
@@ -12,7 +12,6 @@
 #include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/string.h>
 #include <linux/unaligned.h>
 
 asmlinkage void poly1305_block_init_arch(

@@ -35,17 +34,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state,
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
-{
-	dctx->s[0] = get_unaligned_le32(key + 16);
-	dctx->s[1] = get_unaligned_le32(key + 20);
-	dctx->s[2] = get_unaligned_le32(key + 24);
-	dctx->s[3] = get_unaligned_le32(key + 28);
-	dctx->buflen = 0;
-	poly1305_block_init_arch(&dctx->state, key);
-}
-EXPORT_SYMBOL(poly1305_init_arch);
-
 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
 			  unsigned int len, u32 padbit)
 {

@@ -67,51 +55,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
 }
 EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
 
-void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
-			  unsigned int nbytes)
-{
-	if (unlikely(dctx->buflen)) {
-		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
-
-		memcpy(dctx->buf + dctx->buflen, src, bytes);
-		src += bytes;
-		nbytes -= bytes;
-		dctx->buflen += bytes;
-
-		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			poly1305_blocks_arch(&dctx->state, dctx->buf,
-					     POLY1305_BLOCK_SIZE, 1);
-			dctx->buflen = 0;
-		}
-	}
-
-	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
-		poly1305_blocks_arch(&dctx->state, src, nbytes, 1);
-		src += round_down(nbytes, POLY1305_BLOCK_SIZE);
-		nbytes %= POLY1305_BLOCK_SIZE;
-	}
-
-	if (unlikely(nbytes)) {
-		dctx->buflen = nbytes;
-		memcpy(dctx->buf, src, nbytes);
-	}
-}
-EXPORT_SYMBOL(poly1305_update_arch);
-
-void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
-{
-	if (unlikely(dctx->buflen)) {
-		dctx->buf[dctx->buflen++] = 1;
-		memset(dctx->buf + dctx->buflen, 0,
-		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-	}
-
-	poly1305_emit_arch(&dctx->h, dst, dctx->s);
-	*dctx = (struct poly1305_desc_ctx){};
-}
-EXPORT_SYMBOL(poly1305_final_arch);
-
 bool poly1305_is_arch_optimized(void)
 {
 	/* We always can use at least the ARM scalar implementation. */

arch/arm64/lib/crypto/poly1305-glue.c +0 ~~-58~~

76 lines not on this page · raw message

arch/mips/lib/crypto/poly1305-glue.c +0 ~~-60~~

72 lines not on this page · raw message

arch/powerpc/lib/crypto/poly1305-p10-glue.c +0 ~~-63~~

81 lines not on this page · raw message

arch/x86/lib/crypto/poly1305_glue.c +0 ~~-60~~

72 lines not on this page · raw message

include/crypto/poly1305.h +6 ~~-47~~

61 lines not on this page · raw message

lib/crypto/poly1305.c +26 ~~-13~~

index f35692376acf..e2bcce281330 100644
@@ -22,47 +22,60 @@ void poly1305_block_init_generic(struct poly1305_block_state *desc,
 }
 EXPORT_SYMBOL_GPL(poly1305_block_init_generic);
 
-void poly1305_init_generic(struct poly1305_desc_ctx *desc,
-			   const u8 key[POLY1305_KEY_SIZE])
+void poly1305_init(struct poly1305_desc_ctx *desc,
+		   const u8 key[POLY1305_KEY_SIZE])
 {
 	desc->s[0] = get_unaligned_le32(key + 16);
 	desc->s[1] = get_unaligned_le32(key + 20);
 	desc->s[2] = get_unaligned_le32(key + 24);
 	desc->s[3] = get_unaligned_le32(key + 28);
 	desc->buflen = 0;
-	poly1305_block_init_generic(&desc->state, key);
+	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
+		poly1305_block_init_arch(&desc->state, key);
+	else
+		poly1305_block_init_generic(&desc->state, key);
 }
-EXPORT_SYMBOL_GPL(poly1305_init_generic);
+EXPORT_SYMBOL(poly1305_init);
 
 static inline void poly1305_blocks(struct poly1305_block_state *state,
 				   const u8 *src, unsigned int len)
 {
-	poly1305_blocks_generic(state, src, len, 1);
+	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
+		poly1305_blocks_arch(state, src, len, 1);
+	else
+		poly1305_blocks_generic(state, src, len, 1);
 }
 
-void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src,
-			     unsigned int nbytes)
+void poly1305_update(struct poly1305_desc_ctx *desc,
+		     const u8 *src, unsigned int nbytes)
 {
 	desc->buflen = BLOCK_HASH_UPDATE(&poly1305_blocks, &desc->state,
 					 src, nbytes, POLY1305_BLOCK_SIZE,
 					 desc->buf, desc->buflen);
 }
-EXPORT_SYMBOL_GPL(poly1305_update_generic);
+EXPORT_SYMBOL(poly1305_update);
 
-void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst)
+void poly1305_final(struct poly1305_desc_ctx *desc, u8 *dst)
 {
 	if (unlikely(desc->buflen)) {
 		desc->buf[desc->buflen++] = 1;
 		memset(desc->buf + desc->buflen, 0,
 		       POLY1305_BLOCK_SIZE - desc->buflen);
-		poly1305_blocks_generic(&desc->state, desc->buf,
-					POLY1305_BLOCK_SIZE, 0);
+		if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
+			poly1305_blocks_arch(&desc->state, desc->buf,
+					     POLY1305_BLOCK_SIZE, 0);
+		else
+			poly1305_blocks_generic(&desc->state, desc->buf,
+						POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_emit_generic(&desc->h, dst, desc->s);
+	if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305))
+		poly1305_emit_arch(&desc->state.h, dst, desc->s);
+	else
+		poly1305_emit_generic(&desc->state.h, dst, desc->s);
 	*desc = (struct poly1305_desc_ctx){};
 }
-EXPORT_SYMBOL_GPL(poly1305_final_generic);
+EXPORT_SYMBOL(poly1305_final);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Martin Willi <[email protected]>");

Signature · 1 line

2.39.5