crypto: lib - Add partial block helper

new file mode 100644
index 000000000000..4184e2337d68
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Handle partial blocks for block hash.
+ *
+ * Copyright (c) 2015 Linaro Ltd <[email protected]>
+ * Copyright (c) 2025 Herbert Xu <[email protected]>
+ */
+
+#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H
+#define _CRYPTO_INTERNAL_BLOCKHASH_H
+
+#include <linux/string.h>
+#include <linux/types.h>
+
+#define BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, dv, buf,	\
+			       buflen)					\
+	({								\
+		unsigned int _nbytes = (nbytes);			\
+		unsigned int _buflen = (buflen);			\
+		typeof(block) _block = (block);				\
+		typeof(state) _state = (state); 			\
+		unsigned int _bs = (bs);				\
+		unsigned int _dv = (dv);				\
+		const u8 *_src = (src);					\
+		u8 *_buf = (buf);					\
+		while ((_buflen + _nbytes) >= _bs) {			\
+			unsigned int len = _nbytes;			\
+			const u8 *data = _src;				\
+			int blocks, remain;				\
+			if (_buflen) {					\
+				remain = _bs - _buflen;			\
+				memcpy(_buf + _buflen, _src, remain);	\
+				data = _buf;				\
+				len = _bs;				\
+			}						\
+			remain = len % bs;				\
+			blocks = (len - remain) / _dv;			\
+			_block(_state, data, blocks);			\
+			_src += len - remain - _buflen;			\
+			_nbytes -= len - remain - _buflen;		\
+			_buflen = 0;					\
+		}							\
+		memcpy(_buf + _buflen, _src, _nbytes);			\
+		_buflen += _nbytes;					\
+	})
+
+#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen)
+#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \
+	BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen)
+
+#endif	/* _CRYPTO_INTERNAL_BLOCKHASH_H */

index abbd882f7849..f873c2207b1e 100644
@@ -71,8 +71,13 @@ struct crypto_sha256_state {
 };
 
 struct sha256_state {
-	u32 state[SHA256_DIGEST_SIZE / 4];
-	u64 count;
+	union {
+		struct crypto_sha256_state ctx;
+		struct {
+			u32 state[SHA256_DIGEST_SIZE / 4];
+			u64 count;
+		};
+	};
 	u8 buf[SHA256_BLOCK_SIZE];
 };
 

index 08cd5e41d4fd..9f284bed5a51 100644
@@ -8,6 +8,7 @@
 #ifndef _CRYPTO_SHA256_BASE_H
 #define _CRYPTO_SHA256_BASE_H
 
+#include <crypto/internal/blockhash.h>
 #include <crypto/internal/hash.h>
 #include <crypto/sha2.h>
 #include <linux/math.h>

@@ -40,35 +41,10 @@ static inline int lib_sha256_base_do_update(struct sha256_state *sctx,
 					    sha256_block_fn *block_fn)
 {
 	unsigned int partial = sctx->count % SHA256_BLOCK_SIZE;
-	struct crypto_sha256_state *state = (void *)sctx;
 
 	sctx->count += len;
-
-	if (unlikely((partial + len) >= SHA256_BLOCK_SIZE)) {
-		int blocks;
-
-		if (partial) {
-			int p = SHA256_BLOCK_SIZE - partial;
-
-			memcpy(sctx->buf + partial, data, p);
-			data += p;
-			len -= p;
-
-			block_fn(state, sctx->buf, 1);
-		}
-
-		blocks = len / SHA256_BLOCK_SIZE;
-		len %= SHA256_BLOCK_SIZE;
-
-		if (blocks) {
-			block_fn(state, data, blocks);
-			data += blocks * SHA256_BLOCK_SIZE;
-		}
-		partial = 0;
-	}
-	if (len)
-		memcpy(sctx->buf + partial, data, len);
-
+	BLOCK_HASH_UPDATE_BLOCKS(block_fn, &sctx->ctx, data, len,
+				 SHA256_BLOCK_SIZE, sctx->buf, partial);
 	return 0;
 }
 

@@ -140,14 +116,6 @@ static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx,
 	return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn);
 }
 
-static inline int sha256_base_do_finalize(struct shash_desc *desc,
-					  sha256_block_fn *block_fn)
-{
-	struct sha256_state *sctx = shash_desc_ctx(desc);
-
-	return lib_sha256_base_do_finalize(sctx, block_fn);
-}
-
 static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4],
 				       u8 *out, unsigned int digest_size)
 {

index e614594f88c1..e81f752c6cbb 100644
@@ -6,9 +6,8 @@
 #ifndef _CRYPTO_INTERNAL_POLY1305_H
 #define _CRYPTO_INTERNAL_POLY1305_H
 
-#include <linux/unaligned.h>
-#include <linux/types.h>
 #include <crypto/poly1305.h>
+#include <linux/types.h>
 
 /*
  * Poly1305 core functions.  These only accept whole blocks; the caller must

@@ -31,4 +30,29 @@ void poly1305_core_blocks(struct poly1305_state *state,
 void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
 			void *dst);
 
+void poly1305_block_init_arch(struct poly1305_block_state *state,
+			      const u8 key[POLY1305_BLOCK_SIZE]);
+void poly1305_block_init_generic(struct poly1305_block_state *state,
+				 const u8 key[POLY1305_BLOCK_SIZE]);
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+			  unsigned int len, u32 padbit);
+
+static inline void poly1305_blocks_generic(struct poly1305_block_state *state,
+					   const u8 *src, unsigned int len,
+					   u32 padbit)
+{
+	poly1305_core_blocks(&state->h, &state->core_r, src,
+			     len / POLY1305_BLOCK_SIZE, padbit);
+}
+
+void poly1305_emit_arch(const struct poly1305_state *state,
+			u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]);
+
+static inline void poly1305_emit_generic(const struct poly1305_state *state,
+					 u8 digest[POLY1305_DIGEST_SIZE],
+					 const u32 nonce[4])
+{
+	poly1305_core_emit(state, nonce, digest);
+}
+
 #endif

index 6e21ec2d1dc2..027d74842cd5 100644
@@ -7,7 +7,6 @@
 #define _CRYPTO_POLY1305_H
 
 #include <linux/types.h>
-#include <linux/crypto.h>
 
 #define POLY1305_BLOCK_SIZE	16
 #define POLY1305_KEY_SIZE	32

@@ -38,6 +37,17 @@ struct poly1305_state {
 	};
 };
 
+/* Combined state for block function. */
+struct poly1305_block_state {
+	/* accumulator */
+	struct poly1305_state h;
+	/* key */
+	union {
+		struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
+		struct poly1305_core_key core_r;
+	};
+};
+
 struct poly1305_desc_ctx {
 	/* partial buffer */
 	u8 buf[POLY1305_BLOCK_SIZE];

@@ -45,12 +55,15 @@ struct poly1305_desc_ctx {
 	unsigned int buflen;
 	/* finalize key */
 	u32 s[4];
-	/* accumulator */
-	struct poly1305_state h;
-	/* key */
 	union {
-		struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
-		struct poly1305_core_key core_r;
+		struct {
+			struct poly1305_state h;
+			union {
+				struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE];
+				struct poly1305_core_key core_r;
+			};
+		};
+		struct poly1305_block_state state;
 	};
 };
 

87 lines not on this page · raw message

125 lines not on this page · raw message

113 lines not on this page · raw message

index 576e7a58e0b1..fc00e96b2a5b 100644
@@ -5,23 +5,32 @@
  * Copyright (C) 2019 Linaro Ltd. <[email protected]>
  */
 
-#include <crypto/poly1305.h>
+#include <crypto/internal/poly1305.h>
 #include <linux/cpufeature.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
 
-asmlinkage void poly1305_init_mips(void *state, const u8 *key);
-asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit);
-asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce);
+asmlinkage void poly1305_block_init_arch(struct poly1305_block_state *state,
+					 const u8 key[POLY1305_BLOCK_SIZE]);
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+asmlinkage void poly1305_blocks_arch(struct poly1305_block_state *state,
+				     const u8 *src, u32 len, u32 hibit);
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+				   u8 digest[POLY1305_DIGEST_SIZE],
+				   const u32 nonce[4]);
+EXPORT_SYMBOL_GPL(poly1305_emit_arch);
 
 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
 {
-	poly1305_init_mips(&dctx->h, key);
 	dctx->s[0] = get_unaligned_le32(key + 16);
 	dctx->s[1] = get_unaligned_le32(key + 20);
 	dctx->s[2] = get_unaligned_le32(key + 24);
 	dctx->s[3] = get_unaligned_le32(key + 28);
 	dctx->buflen = 0;
+	poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 

@@ -37,7 +46,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
 		dctx->buflen += bytes;
 
 		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			poly1305_blocks_mips(&dctx->h, dctx->buf,
+			poly1305_blocks_arch(&dctx->state, dctx->buf,
 					     POLY1305_BLOCK_SIZE, 1);
 			dctx->buflen = 0;
 		}

@@ -46,7 +55,7 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
 	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
 		unsigned int len = round_down(nbytes, POLY1305_BLOCK_SIZE);
 
-		poly1305_blocks_mips(&dctx->h, src, len, 1);
+		poly1305_blocks_arch(&dctx->state, src, len, 1);
 		src += len;
 		nbytes %= POLY1305_BLOCK_SIZE;
 	}

@@ -64,10 +73,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_emit_mips(&dctx->h, dst, dctx->s);
+	poly1305_emit_arch(&dctx->h, dst, dctx->s);
 	*dctx = (struct poly1305_desc_ctx){};
 }
 EXPORT_SYMBOL(poly1305_final_arch);

index 00617f4c58e6..a33c61efd360 100644
@@ -4,19 +4,20 @@
  *
  * Copyright 2023- IBM Corp. All rights reserved.
  */
+#include <asm/switch_to.h>
+#include <crypto/internal/poly1305.h>
+#include <linux/cpufeature.h>
+#include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/jump_label.h>
-#include <crypto/internal/simd.h>
-#include <crypto/poly1305.h>
-#include <linux/cpufeature.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
-#include <asm/simd.h>
-#include <asm/switch_to.h>
 
-asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen);
-asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit);
-asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst);
+asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
+asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit);
+asmlinkage void poly1305_emit_arch(const struct poly1305_state *state,
+				   u8 digest[POLY1305_DIGEST_SIZE],
+				   const u32 nonce[4]);
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
 

@@ -32,22 +33,49 @@ static void vsx_end(void)
 	preempt_enable();
 }
 
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+void poly1305_block_init_arch(struct poly1305_block_state *dctx,
+			      const u8 key[POLY1305_BLOCK_SIZE])
 {
 	if (!static_key_enabled(&have_p10))
-		return poly1305_init_generic(dctx, key);
+		return poly1305_block_init_generic(dctx, key);
 
 	dctx->h = (struct poly1305_state){};
 	dctx->core_r.key.r64[0] = get_unaligned_le64(key + 0);
 	dctx->core_r.key.r64[1] = get_unaligned_le64(key + 8);
+}
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
+
+void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
+{
 	dctx->s[0] = get_unaligned_le32(key + 16);
 	dctx->s[1] = get_unaligned_le32(key + 20);
 	dctx->s[2] = get_unaligned_le32(key + 24);
 	dctx->s[3] = get_unaligned_le32(key + 28);
 	dctx->buflen = 0;
+	poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
+			  unsigned int len, u32 padbit)
+{
+	if (!static_key_enabled(&have_p10))
+		return poly1305_blocks_generic(state, src, len, padbit);
+	vsx_begin();
+	if (len >= POLY1305_BLOCK_SIZE * 4) {
+		poly1305_p10le_4blocks(state, src, len);
+		src += len - (len % (POLY1305_BLOCK_SIZE * 4));
+		len %= POLY1305_BLOCK_SIZE * 4;
+	}
+	while (len >= POLY1305_BLOCK_SIZE) {
+		poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit);
+		len -= POLY1305_BLOCK_SIZE;
+		src += POLY1305_BLOCK_SIZE;
+	}
+	vsx_end();
+}
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
+
 void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
 			  const u8 *src, unsigned int srclen)
 {

@@ -64,28 +92,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx,
 		dctx->buflen += bytes;
 		if (dctx->buflen < POLY1305_BLOCK_SIZE)
 			return;
-		vsx_begin();
-		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
-		vsx_end();
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 1);
 		dctx->buflen = 0;
 	}
 
 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
-		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-		if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) {
-			vsx_begin();
-			poly1305_p10le_4blocks(&dctx->h, src, srclen);
-			vsx_end();
-			src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4));
-			srclen %= POLY1305_BLOCK_SIZE * 4;
-		}
-		while (srclen >= POLY1305_BLOCK_SIZE) {
-			vsx_begin();
-			poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1);
-			vsx_end();
-			srclen -= POLY1305_BLOCK_SIZE;
-			src += POLY1305_BLOCK_SIZE;
-		}
+		poly1305_blocks_arch(&dctx->state, src, srclen, 1);
+		src += srclen - (srclen % POLY1305_BLOCK_SIZE);
+		srclen %= POLY1305_BLOCK_SIZE;
 	}
 
 	if (unlikely(srclen)) {

@@ -104,12 +119,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		vsx_begin();
-		poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-		vsx_end();
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_emit_64(&dctx->h, &dctx->s, dst);
+	poly1305_emit_arch(&dctx->h, dst, dctx->s);
 }
 EXPORT_SYMBOL(poly1305_final_arch);
 

index cff35ca5822a..b45818c51223 100644
@@ -3,34 +3,15 @@
  * Copyright (C) 2015-2019 Jason A. Donenfeld <[email protected]>. All Rights Reserved.
  */
 
-#include <crypto/internal/simd.h>
-#include <crypto/poly1305.h>
+#include <asm/cpu_device_id.h>
+#include <asm/fpu/api.h>
+#include <crypto/internal/poly1305.h>
 #include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/sizes.h>
+#include <linux/string.h>
 #include <linux/unaligned.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
-
-asmlinkage void poly1305_init_x86_64(void *ctx,
-				     const u8 key[POLY1305_BLOCK_SIZE]);
-asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
-				       const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-				     const u32 nonce[4]);
-asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-				  const u32 nonce[4]);
-asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
-				    const u32 padbit);
-asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
-				     const u32 padbit);
-asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
-				       const size_t len, const u32 padbit);
-
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
-static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);
 
 struct poly1305_arch_internal {
 	union {

@@ -45,64 +26,55 @@ struct poly1305_arch_internal {
 	struct { u32 r2, r1, r4, r3; } rn[9];
 };
 
-/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit
- * the unfortunate situation of using AVX and then having to go back to scalar
- * -- because the user is silly and has called the update function from two
- * separate contexts -- then we need to convert back to the original base before
- * proceeding. It is possible to reason that the initial reduction below is
- * sufficient given the implementation invariants. However, for an avoidance of
- * doubt and because this is not performance critical, we do the full reduction
- * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py
- */
-static void convert_to_base2_64(void *ctx)
+asmlinkage void poly1305_init_x86_64(struct poly1305_block_state *state,
+				     const u8 key[POLY1305_BLOCK_SIZE]);
+asmlinkage void poly1305_blocks_x86_64(struct poly1305_arch_internal *ctx,
+				       const u8 *inp,
+				       const size_t len, const u32 padbit);
+asmlinkage void poly1305_emit_x86_64(const struct poly1305_state *ctx,
+				     u8 mac[POLY1305_DIGEST_SIZE],
+				     const u32 nonce[4]);
+asmlinkage void poly1305_emit_avx(const struct poly1305_state *ctx,
+				  u8 mac[POLY1305_DIGEST_SIZE],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_blocks_avx(struct poly1305_arch_internal *ctx,
+				    const u8 *inp, const size_t len,
+				    const u32 padbit);
+asmlinkage void poly1305_blocks_avx2(struct poly1305_arch_internal *ctx,
+				     const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_blocks_avx512(struct poly1305_arch_internal *ctx,
+				       const u8 *inp,
+				       const size_t len, const u32 padbit);
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512);
+
+void poly1305_block_init_arch(struct poly1305_block_state *state,
+			      const u8 key[POLY1305_BLOCK_SIZE])
 {
-	struct poly1305_arch_internal *state = ctx;
-	u32 cy;
-
-	if (!state->is_base2_26)
-		return;
-
-	cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
-	cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
-	cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
-	cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
-	state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
-	state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
-	state->hs[2] = state->h[4] >> 24;
-#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
-	cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL);
-	state->hs[2] &= 3;
-	state->hs[0] += cy;
-	state->hs[1] += (cy = ULT(state->hs[0], cy));
-	state->hs[2] += ULT(state->hs[1], cy);
-#undef ULT
-	state->is_base2_26 = 0;
+	poly1305_init_x86_64(state, key);
 }
+EXPORT_SYMBOL_GPL(poly1305_block_init_arch);
 
-static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_BLOCK_SIZE])
+void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *inp,
+			  unsigned int len, u32 padbit)
 {
-	poly1305_init_x86_64(ctx, key);
-}
-
-static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len,
-				 const u32 padbit)
-{
-	struct poly1305_arch_internal *state = ctx;
+	struct poly1305_arch_internal *ctx =
+		container_of(&state->h.h, struct poly1305_arch_internal, h);
 
 	/* SIMD disables preemption, so relax after processing each page. */
 	BUILD_BUG_ON(SZ_4K < POLY1305_BLOCK_SIZE ||
 		     SZ_4K % POLY1305_BLOCK_SIZE);
 
-	if (!static_branch_likely(&poly1305_use_avx) ||
-	    (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) ||
-	    !crypto_simd_usable()) {
-		convert_to_base2_64(ctx);
+	if (!static_branch_likely(&poly1305_use_avx)) {
 		poly1305_blocks_x86_64(ctx, inp, len, padbit);
 		return;
 	}
 
 	do {
-		const size_t bytes = min_t(size_t, len, SZ_4K);
+		const unsigned int bytes = min(len, SZ_4K);
 
 		kernel_fpu_begin();
 		if (static_branch_likely(&poly1305_use_avx512))

@@ -117,24 +89,26 @@ static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len,
 		inp += bytes;
 	} while (len);
 }
+EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
 
-static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE],
-			       const u32 nonce[4])
+void poly1305_emit_arch(const struct poly1305_state *ctx,
+			u8 mac[POLY1305_DIGEST_SIZE], const u32 nonce[4])
 {
 	if (!static_branch_likely(&poly1305_use_avx))
 		poly1305_emit_x86_64(ctx, mac, nonce);
 	else
 		poly1305_emit_avx(ctx, mac, nonce);
 }
+EXPORT_SYMBOL_GPL(poly1305_emit_arch);
 
 void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
 {
-	poly1305_simd_init(&dctx->h, key);
 	dctx->s[0] = get_unaligned_le32(&key[16]);
 	dctx->s[1] = get_unaligned_le32(&key[20]);
 	dctx->s[2] = get_unaligned_le32(&key[24]);
 	dctx->s[3] = get_unaligned_le32(&key[28]);
 	dctx->buflen = 0;
+	poly1305_block_init_arch(&dctx->state, key);
 }
 EXPORT_SYMBOL(poly1305_init_arch);
 

@@ -151,14 +125,15 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
 		dctx->buflen += bytes;
 
 		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1);
+			poly1305_blocks_arch(&dctx->state, dctx->buf,
+					     POLY1305_BLOCK_SIZE, 1);
 			dctx->buflen = 0;
 		}
 	}
 
 	if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
 		bytes = round_down(srclen, POLY1305_BLOCK_SIZE);
-		poly1305_simd_blocks(&dctx->h, src, bytes, 1);
+		poly1305_blocks_arch(&dctx->state, src, bytes, 1);
 		src += bytes;
 		srclen -= bytes;
 	}

@@ -176,10 +151,11 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
 		dctx->buf[dctx->buflen++] = 1;
 		memset(dctx->buf + dctx->buflen, 0,
 		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0);
+		poly1305_blocks_arch(&dctx->state, dctx->buf,
+				     POLY1305_BLOCK_SIZE, 0);
 	}
 
-	poly1305_simd_emit(&dctx->h, dst, dctx->s);
+	poly1305_emit_arch(&dctx->h, dst, dctx->s);
 	memzero_explicit(dctx, sizeof(*dctx));
 }
 EXPORT_SYMBOL(poly1305_final_arch);

index e0436bdc462b..9c36ff4bd9c4 100644
@@ -9,57 +9,107 @@
  * (at your option) any later version.
  */
 
-#include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/poly1305.h>
-#include <linux/crypto.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
+
+#define CRYPTO_POLY1305_TFM_HAS_KEY 0x00100000
+
+struct crypto_poly1305_ctx {
+	struct poly1305_block_state state;
+	u32 s[4];
+};
 
 struct crypto_poly1305_desc_ctx {
-	struct poly1305_desc_ctx base;
-	u8 key[POLY1305_KEY_SIZE];
+	struct crypto_poly1305_ctx base;
 	unsigned int keysize;
 };
 
+static int crypto_poly1305_setkey(struct crypto_shash *tfm, const u8 *key,
+				  unsigned int len, bool arch)
+{
+	struct crypto_poly1305_ctx *ctx = crypto_shash_ctx(tfm);
+
+	if (len != POLY1305_KEY_SIZE)
+		return -EINVAL;
+
+	if (arch)
+		poly1305_block_init_arch(&ctx->state, key);
+	else
+		poly1305_block_init_generic(&ctx->state, key);
+	memcpy(ctx->s, key + POLY1305_BLOCK_SIZE, sizeof(ctx->s));
+	crypto_shash_set_flags(tfm, CRYPTO_POLY1305_TFM_HAS_KEY);
+	return 0;
+}
+
+static int crypto_poly1305_setkey_generic(struct crypto_shash *tfm,
+					  const u8 *key, unsigned int len)
+{
+	return crypto_poly1305_setkey(tfm, key, len, false);
+}
+
+static int crypto_poly1305_setkey_arch(struct crypto_shash *tfm, const u8 *key,
+				       unsigned int len)
+{
+	return crypto_poly1305_setkey(tfm, key, len, true);
+}
+
 static int crypto_poly1305_init(struct shash_desc *desc)
 {
 	struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
+	struct crypto_shash *tfm = desc->tfm;
+	struct crypto_poly1305_ctx *ctx;
 
+	ctx = crypto_shash_ctx(tfm);
+	dctx->base = *ctx;
 	dctx->keysize = 0;
+	if (crypto_shash_get_flags(tfm) & CRYPTO_POLY1305_TFM_HAS_KEY)
+		dctx->keysize = POLY1305_KEY_SIZE;
 	return 0;
 }
 
-static int crypto_poly1305_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, bool arch)
+static inline int crypto_poly1305_update(struct shash_desc *desc,
+					 const u8 *src, unsigned int srclen,
+					 bool arch)
 {
 	struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-	unsigned int bytes;
 
 	/*
 	 * The key is passed as the first 32 "data" bytes.  The actual
 	 * poly1305_init() can be called only once the full key is available.
 	 */
 	if (dctx->keysize < POLY1305_KEY_SIZE) {
-		bytes = min(srclen, POLY1305_KEY_SIZE - dctx->keysize);
-		memcpy(&dctx->key[dctx->keysize], src, bytes);
-		dctx->keysize += bytes;
-		if (dctx->keysize < POLY1305_KEY_SIZE)
-			return 0;
-		if (arch)
-			poly1305_init(&dctx->base, dctx->key);
-		else
-			poly1305_init_generic(&dctx->base, dctx->key);
-		src += bytes;
-		srclen -= bytes;
+		if (!dctx->keysize) {
+			if (arch)
+				poly1305_block_init_arch(&dctx->base.state,
+							 src);
+			else
+				poly1305_block_init_generic(&dctx->base.state,
+							    src);
+		}
+		dctx->keysize += POLY1305_BLOCK_SIZE;
+		src += POLY1305_BLOCK_SIZE;
+		srclen -= POLY1305_BLOCK_SIZE;
+		if (srclen < POLY1305_BLOCK_SIZE)
+			return srclen;
+
+		memcpy(&dctx->base.s, src, POLY1305_BLOCK_SIZE);
+		dctx->keysize += POLY1305_BLOCK_SIZE;
+		src += POLY1305_BLOCK_SIZE;
+		srclen -= POLY1305_BLOCK_SIZE;
+		if (srclen < POLY1305_BLOCK_SIZE)
+			return srclen;
 	}
 
 	if (arch)
-		poly1305_update(&dctx->base, src, srclen);
+		poly1305_blocks_arch(&dctx->base.state, src, srclen, 1);
 	else
-		poly1305_update_generic(&dctx->base, src, srclen);
+		poly1305_blocks_generic(&dctx->base.state, src, srclen, 1);
 
-	return 0;
+	return srclen % POLY1305_BLOCK_SIZE;
 }
 
 static int crypto_poly1305_update_generic(struct shash_desc *desc,

@@ -74,29 +124,46 @@ static int crypto_poly1305_update_arch(struct shash_desc *desc,
 	return crypto_poly1305_update(desc, src, srclen, true);
 }
 
-static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst, bool arch)
+static inline int crypto_poly1305_finup(struct shash_desc *desc, const u8 *src,
+					unsigned int len, u8 *dst, bool arch)
 {
 	struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
 
 	if (unlikely(dctx->keysize != POLY1305_KEY_SIZE))
 		return -ENOKEY;
 
+	if (unlikely(len)) {
+		u8 block[POLY1305_BLOCK_SIZE] = {};
+
+		memcpy(block, src, len);
+		block[len] = 1;
+		if (arch)
+			poly1305_blocks_arch(&dctx->base.state, block,
+					     POLY1305_BLOCK_SIZE, 0);
+		else
+			poly1305_blocks_generic(&dctx->base.state, block,
+						POLY1305_BLOCK_SIZE, 0);
+		memzero_explicit(block, sizeof(block));
+	}
+
 	if (arch)
-		poly1305_final(&dctx->base, dst);
+		poly1305_emit_arch(&dctx->base.state.h, dst, dctx->base.s);
 	else
-		poly1305_final_generic(&dctx->base, dst);
-	memzero_explicit(&dctx->key, sizeof(dctx->key));
+		poly1305_emit_generic(&dctx->base.state.h, dst, dctx->base.s);
 	return 0;
 }
 
-static int crypto_poly1305_final_generic(struct shash_desc *desc, u8 *dst)
+static int crypto_poly1305_finup_generic(struct shash_desc *desc,
+					 const u8 *src, unsigned int len,
+					 u8 *dst)
 {
-	return crypto_poly1305_final(desc, dst, false);
+	return crypto_poly1305_finup(desc, src, len, dst, false);
 }
 
-static int crypto_poly1305_final_arch(struct shash_desc *desc, u8 *dst)
+static int crypto_poly1305_finup_arch(struct shash_desc *desc,
+				      const u8 *src, unsigned int len, u8 *dst)
 {
-	return crypto_poly1305_final(desc, dst, true);
+	return crypto_poly1305_finup(desc, src, len, dst, true);
 }
 
 static struct shash_alg poly1305_algs[] = {

@@ -104,24 +171,32 @@ static struct shash_alg poly1305_algs[] = {
 		.base.cra_name		= "poly1305",
 		.base.cra_driver_name	= "poly1305-generic",
 		.base.cra_priority	= 100,
+		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
+					  CRYPTO_ALG_OPTIONAL_KEY,
 		.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
+		.base.cra_ctxsize	= sizeof(struct crypto_poly1305_ctx),
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= POLY1305_DIGEST_SIZE,
 		.init			= crypto_poly1305_init,
+		.setkey			= crypto_poly1305_setkey_generic,
 		.update			= crypto_poly1305_update_generic,
-		.final			= crypto_poly1305_final_generic,
+		.finup			= crypto_poly1305_finup_generic,
 		.descsize		= sizeof(struct crypto_poly1305_desc_ctx),
 	},
 	{
 		.base.cra_name		= "poly1305",
 		.base.cra_driver_name	= "poly1305-" __stringify(ARCH),
 		.base.cra_priority	= 300,
+		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
+					  CRYPTO_ALG_OPTIONAL_KEY,
 		.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
+		.base.cra_ctxsize	= sizeof(struct crypto_poly1305_ctx),
 		.base.cra_module	= THIS_MODULE,
 		.digestsize		= POLY1305_DIGEST_SIZE,
 		.init			= crypto_poly1305_init,
+		.setkey			= crypto_poly1305_setkey_arch,
 		.update			= crypto_poly1305_update_arch,
-		.final			= crypto_poly1305_final_arch,
+		.finup			= crypto_poly1305_finup_arch,
 		.descsize		= sizeof(struct crypto_poly1305_desc_ctx),
 	},
 };

index fe75320ff9a3..43ede1958a18 100644
@@ -140,9 +140,12 @@
 
 /*
  * Transform masks and values (for crt_flags).
+ *
+ * The top three nibbles 0xfff00000 are reserved for the algorithm.
  */
 #define CRYPTO_TFM_NEED_KEY		0x00000001
 
+/* The top three nibbles 0xfff00000 are reserved for the algorithm. */
 #define CRYPTO_TFM_REQ_MASK		0x000fff00
 #define CRYPTO_TFM_REQ_FORBID_WEAK_KEYS	0x00000100
 #define CRYPTO_TFM_REQ_MAY_SLEEP	0x00000200

index e9082d0d9e99..aa7402523f41 100644
@@ -12,7 +12,6 @@
 #include <linux/jump_label.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/string.h>
 #include <linux/unaligned.h>
 
 asmlinkage void poly1305_block_init_arch(struct poly1305_block_state *state,

@@ -34,17 +33,6 @@ void __weak poly1305_blocks_neon(struct poly1305_block_state *state,
 
 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
 
-void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 key[POLY1305_KEY_SIZE])
-{
-	dctx->s[0] = get_unaligned_le32(key + 16);
-	dctx->s[1] = get_unaligned_le32(key + 20);
-	dctx->s[2] = get_unaligned_le32(key + 24);
-	dctx->s[3] = get_unaligned_le32(key + 28);
-	dctx->buflen = 0;
-	poly1305_block_init_arch(&dctx->state, key);
-}
-EXPORT_SYMBOL(poly1305_init_arch);
-
 void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
 			  unsigned int len, u32 padbit)
 {

@@ -66,51 +54,6 @@ void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src,
 }
 EXPORT_SYMBOL_GPL(poly1305_blocks_arch);
 
-void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
-			  unsigned int nbytes)
-{
-	if (unlikely(dctx->buflen)) {
-		u32 bytes = min(nbytes, POLY1305_BLOCK_SIZE - dctx->buflen);
-
-		memcpy(dctx->buf + dctx->buflen, src, bytes);
-		src += bytes;
-		nbytes -= bytes;
-		dctx->buflen += bytes;
-
-		if (dctx->buflen == POLY1305_BLOCK_SIZE) {
-			poly1305_blocks_arch(&dctx->state, dctx->buf,
-					     POLY1305_BLOCK_SIZE, 1);
-			dctx->buflen = 0;
-		}
-	}
-
-	if (likely(nbytes >= POLY1305_BLOCK_SIZE)) {
-		poly1305_blocks_arch(&dctx->state, src, nbytes, 1);
-		src += round_down(nbytes, POLY1305_BLOCK_SIZE);
-		nbytes %= POLY1305_BLOCK_SIZE;
-	}
-
-	if (unlikely(nbytes)) {
-		dctx->buflen = nbytes;
-		memcpy(dctx->buf, src, nbytes);
-	}
-}
-EXPORT_SYMBOL(poly1305_update_arch);
-
-void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst)
-{
-	if (unlikely(dctx->buflen)) {
-		dctx->buf[dctx->buflen++] = 1;
-		memset(dctx->buf + dctx->buflen, 0,
-		       POLY1305_BLOCK_SIZE - dctx->buflen);
-		poly1305_blocks_arch(&dctx->state, dctx->buf, POLY1305_BLOCK_SIZE, 0);
-	}
-
-	poly1305_emit_arch(&dctx->h, dst, dctx->s);
-	*dctx = (struct poly1305_desc_ctx){};
-}
-EXPORT_SYMBOL(poly1305_final_arch);
-
 bool poly1305_is_arch_optimized(void)
 {
 	/* We always can use at least the ARM scalar implementation. */

76 lines not on this page · raw message

72 lines not on this page · raw message

81 lines not on this page · raw message

72 lines not on this page · raw message

61 lines not on this page · raw message

74 lines not on this page · raw message

index d740849f1c19..cd8e3b5c55b5 100644
@@ -51,6 +51,8 @@ struct chacha_req {
 struct chachapoly_req_ctx {
 	struct scatterlist src[2];
 	struct scatterlist dst[2];
+	/* Poly1305 transform with generated key */
+	struct crypto_ahash *poly;
 	/* the key we generate for Poly1305 using Chacha20 */
 	u8 key[POLY1305_KEY_SIZE];
 	/* calculated Poly1305 tag */

@@ -81,6 +83,26 @@ static inline void async_done_continue(struct aead_request *req, int err,
 		aead_request_complete(req, err);
 }
 
+static inline void poly_done_continue(struct aead_request *req, int err,
+				      int (*cont)(struct aead_request *))
+{
+	if (err && err != -EINPROGRESS && err != -EBUSY) {
+		struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+
+		crypto_free_ahash(rctx->poly);
+	}
+	async_done_continue(req, err, cont);
+}
+
+static int poly_check_err(struct aead_request *req, int err)
+{
+	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+
+	if (err && err != -EINPROGRESS && err != -EBUSY)
+		crypto_free_ahash(rctx->poly);
+	return err;
+}
+
 static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)
 {
 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));

@@ -155,6 +177,8 @@ static int poly_tail_continue(struct aead_request *req)
 {
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 
+	crypto_free_ahash(rctx->poly);
+
 	if (rctx->cryptlen == req->cryptlen) /* encrypting */
 		return poly_copy_tag(req);
 

@@ -163,13 +187,11 @@ static int poly_tail_continue(struct aead_request *req)
 
 static void poly_tail_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_tail_continue);
+	poly_done_continue(data, err, poly_tail_continue);
 }
 
 static int poly_tail(struct aead_request *req)
 {
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	int err;

@@ -180,11 +202,12 @@ static int poly_tail(struct aead_request *req)
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_tail_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 	ahash_request_set_crypt(&preq->req, preq->src,
 				rctx->tag, sizeof(preq->tail));
 
 	err = crypto_ahash_finup(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 

@@ -193,12 +216,11 @@ static int poly_tail(struct aead_request *req)
 
 static void poly_cipherpad_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_tail);
+	poly_done_continue(data, err, poly_tail);
 }
 
 static int poly_cipherpad(struct aead_request *req)
 {
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	unsigned int padlen;

@@ -210,10 +232,11 @@ static int poly_cipherpad(struct aead_request *req)
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_cipherpad_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 	ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
 
 	err = crypto_ahash_update(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 

@@ -222,12 +245,11 @@ static int poly_cipherpad(struct aead_request *req)
 
 static void poly_cipher_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_cipherpad);
+	poly_done_continue(data, err, poly_cipherpad);
 }
 
 static int poly_cipher(struct aead_request *req)
 {
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	struct scatterlist *crypt = req->src;

@@ -240,10 +262,11 @@ static int poly_cipher(struct aead_request *req)
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_cipher_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 	ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
 
 	err = crypto_ahash_update(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 

@@ -252,12 +275,11 @@ static int poly_cipher(struct aead_request *req)
 
 static void poly_adpad_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_cipher);
+	poly_done_continue(data, err, poly_cipher);
 }
 
 static int poly_adpad(struct aead_request *req)
 {
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	unsigned int padlen;

@@ -269,10 +291,11 @@ static int poly_adpad(struct aead_request *req)
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_adpad_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 	ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
 
 	err = crypto_ahash_update(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 

@@ -281,80 +304,76 @@ static int poly_adpad(struct aead_request *req)
 
 static void poly_ad_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_adpad);
+	poly_done_continue(data, err, poly_adpad);
 }
 
 static int poly_ad(struct aead_request *req)
 {
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	int err;
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_ad_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 	ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
 
 	err = crypto_ahash_update(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 
 	return poly_adpad(req);
 }
 
-static void poly_setkey_done(void *data, int err)
-{
-	async_done_continue(data, err, poly_ad);
-}
-
-static int poly_setkey(struct aead_request *req)
-{
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
-	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
-	struct poly_req *preq = &rctx->u.poly;
-	int err;
-
-	sg_init_one(preq->src, rctx->key, sizeof(rctx->key));
-
-	ahash_request_set_callback(&preq->req, rctx->flags,
-				   poly_setkey_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
-	ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
-
-	err = crypto_ahash_update(&preq->req);
-	if (err)
-		return err;
-
-	return poly_ad(req);
-}
-
 static void poly_init_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_setkey);
+	poly_done_continue(data, err, poly_ad);
 }
 
 static int poly_init(struct aead_request *req)
 {
-	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 	struct poly_req *preq = &rctx->u.poly;
 	int err;
 
 	ahash_request_set_callback(&preq->req, rctx->flags,
 				   poly_init_done, req);
-	ahash_request_set_tfm(&preq->req, ctx->poly);
+	ahash_request_set_tfm(&preq->req, rctx->poly);
 
 	err = crypto_ahash_init(&preq->req);
+	err = poly_check_err(req, err);
 	if (err)
 		return err;
 
-	return poly_setkey(req);
+	return poly_ad(req);
+}
+
+static int poly_setkey(struct aead_request *req)
+{
+	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+	struct crypto_ahash *poly;
+	int err;
+
+	poly = crypto_clone_ahash(ctx->poly);
+	if (IS_ERR(poly))
+		return PTR_ERR(poly);
+
+	err = crypto_ahash_setkey(poly, rctx->key, sizeof(rctx->key));
+	if (err) {
+		crypto_free_ahash(poly);
+		return err;
+	}
+
+	rctx->poly = poly;
+
+	return poly_init(req);
 }
 
 static void poly_genkey_done(void *data, int err)
 {
-	async_done_continue(data, err, poly_init);
+	async_done_continue(data, err, poly_setkey);
 }
 
 static int poly_genkey(struct aead_request *req)

@@ -388,7 +407,7 @@ static int poly_genkey(struct aead_request *req)
 	if (err)
 		return err;
 
-	return poly_init(req);
+	return poly_setkey(req);
 }
 
 static void chacha_encrypt_done(void *data, int err)

index afc10af59b0a..09db05b90b5c 100644
@@ -8842,6 +8842,11 @@ static const struct hash_testvec hmac_sha3_512_tv_template[] = {
 
 static const struct hash_testvec poly1305_tv_template[] = {
 	{ /* Test Vector #1 */
+		.key		= "\x00\x00\x00\x00\x00\x00\x00\x00"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
 		.plaintext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"

@@ -8849,20 +8854,17 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.psize		= 96,
+		.psize		= 64,
 		.digest		= "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #2 */
-		.plaintext	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70"
-				  "\xf0\xef\xca\x96\x22\x7a\x86\x3e"
-				  "\x41\x6e\x79\x20\x73\x75\x62\x6d"
+				  "\xf0\xef\xca\x96\x22\x7a\x86\x3e",
+		.ksize		= 32,
+		.plaintext	= "\x41\x6e\x79\x20\x73\x75\x62\x6d"
 				  "\x69\x73\x73\x69\x6f\x6e\x20\x74"
 				  "\x6f\x20\x74\x68\x65\x20\x49\x45"
 				  "\x54\x46\x20\x69\x6e\x74\x65\x6e"

@@ -8909,15 +8911,16 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\x20\x77\x68\x69\x63\x68\x20\x61"
 				  "\x72\x65\x20\x61\x64\x64\x72\x65"
 				  "\x73\x73\x65\x64\x20\x74\x6f",
-		.psize		= 407,
+		.psize		= 375,
 		.digest		= "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70"
 				  "\xf0\xef\xca\x96\x22\x7a\x86\x3e",
 	}, { /* Test Vector #3 */
-		.plaintext	= "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70"
+		.key		= "\x36\xe5\xf6\xb5\xc5\xe0\x60\x70"
 				  "\xf0\xef\xca\x96\x22\x7a\x86\x3e"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x41\x6e\x79\x20\x73\x75\x62\x6d"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\x41\x6e\x79\x20\x73\x75\x62\x6d"
 				  "\x69\x73\x73\x69\x6f\x6e\x20\x74"
 				  "\x6f\x20\x74\x68\x65\x20\x49\x45"
 				  "\x54\x46\x20\x69\x6e\x74\x65\x6e"

@@ -8964,15 +8967,16 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\x20\x77\x68\x69\x63\x68\x20\x61"
 				  "\x72\x65\x20\x61\x64\x64\x72\x65"
 				  "\x73\x73\x65\x64\x20\x74\x6f",
-		.psize		= 407,
+		.psize		= 375,
 		.digest		= "\xf3\x47\x7e\x7c\xd9\x54\x17\xaf"
 				  "\x89\xa6\xb8\x79\x4c\x31\x0c\xf0",
 	}, { /* Test Vector #4 */
-		.plaintext	= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
+		.key		= "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a"
 				  "\xf3\x33\x88\x86\x04\xf6\xb5\xf0"
 				  "\x47\x39\x17\xc1\x40\x2b\x80\x09"
-				  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0"
-				  "\x27\x54\x77\x61\x73\x20\x62\x72"
+				  "\x9d\xca\x5c\xbc\x20\x70\x75\xc0",
+		.ksize		= 32,
+		.plaintext	= "\x27\x54\x77\x61\x73\x20\x62\x72"
 				  "\x69\x6c\x6c\x69\x67\x2c\x20\x61"
 				  "\x6e\x64\x20\x74\x68\x65\x20\x73"
 				  "\x6c\x69\x74\x68\x79\x20\x74\x6f"

@@ -8988,73 +8992,79 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\x68\x65\x20\x6d\x6f\x6d\x65\x20"
 				  "\x72\x61\x74\x68\x73\x20\x6f\x75"
 				  "\x74\x67\x72\x61\x62\x65\x2e",
-		.psize		= 159,
+		.psize		= 127,
 		.digest		= "\x45\x41\x66\x9a\x7e\xaa\xee\x61"
 				  "\xe7\x08\xdc\x7c\xbc\xc5\xeb\x62",
 	}, { /* Test Vector #5 */
-		.plaintext	= "\x02\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x02\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.psize		= 48,
+		.psize		= 16,
 		.digest		= "\x03\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #6 */
-		.plaintext	= "\x02\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x02\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\x02\x00\x00\x00\x00\x00\x00\x00"
+				  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.ksize		= 32,
+		.plaintext	= "\x02\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.psize		= 48,
+		.psize		= 16,
 		.digest		= "\x03\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #7 */
-		.plaintext	= "\x01\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xf0\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\x11\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.psize		= 80,
+		.psize		= 48,
 		.digest		= "\x05\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #8 */
-		.plaintext	= "\x01\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xfb\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
 				  "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe"
 				  "\x01\x01\x01\x01\x01\x01\x01\x01"
 				  "\x01\x01\x01\x01\x01\x01\x01\x01",
-		.psize		= 80,
+		.psize		= 48,
 		.digest		= "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #9 */
-		.plaintext	= "\x02\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x02\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xfd\xff\xff\xff\xff\xff\xff\xff"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xfd\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff",
-		.psize		= 48,
+		.psize		= 16,
 		.digest		= "\xfa\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff",
 	}, { /* Test Vector #10 */
-		.plaintext	= "\x01\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x04\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xe3\x35\x94\xd7\x50\x5e\x43\xb9"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xe3\x35\x94\xd7\x50\x5e\x43\xb9"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x33\x94\xd7\x50\x5e\x43\x79\xcd"
 				  "\x01\x00\x00\x00\x00\x00\x00\x00"

@@ -9062,24 +9072,30 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.psize		= 96,
+		.psize		= 64,
 		.digest		= "\x14\x00\x00\x00\x00\x00\x00\x00"
 				  "\x55\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Test Vector #11 */
-		.plaintext	= "\x01\x00\x00\x00\x00\x00\x00\x00"
+		.key		= "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x04\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\x00\x00\x00\x00\x00\x00\x00\x00"
-				  "\xe3\x35\x94\xd7\x50\x5e\x43\xb9"
+				  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.ksize		= 32,
+		.plaintext	= "\xe3\x35\x94\xd7\x50\x5e\x43\xb9"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x33\x94\xd7\x50\x5e\x43\x79\xcd"
 				  "\x01\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
-		.psize		= 80,
+		.psize		= 48,
 		.digest		= "\x13\x00\x00\x00\x00\x00\x00\x00"
 				  "\x00\x00\x00\x00\x00\x00\x00\x00",
 	}, { /* Regression test for overflow in AVX2 implementation */
+		.key		= "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\xff\xff\xff\xff\xff\xff\xff\xff"
+				  "\xff\xff\xff\xff\xff\xff\xff\xff",
+		.ksize		= 32,
 		.plaintext	= "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"

@@ -9113,12 +9129,8 @@ static const struct hash_testvec poly1305_tv_template[] = {
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
-				  "\xff\xff\xff\xff\xff\xff\xff\xff"
 				  "\xff\xff\xff\xff",
-		.psize		= 300,
+		.psize		= 268,
 		.digest		= "\xfb\x5e\x96\xd8\x61\xd5\xc7\xc8"
 				  "\x78\xe5\x87\xcc\x2d\x5a\x22\xe1",
 	}

index 9c36ff4bd9c4..5fdf36b2e028 100644
@@ -24,8 +24,7 @@ struct crypto_poly1305_ctx {
 };
 
 struct crypto_poly1305_desc_ctx {
-	struct crypto_poly1305_ctx base;
-	unsigned int keysize;
+	struct poly1305_block_state state;
 };
 
 static int crypto_poly1305_setkey(struct crypto_shash *tfm, const u8 *key,

@@ -64,10 +63,7 @@ static int crypto_poly1305_init(struct shash_desc *desc)
 	struct crypto_poly1305_ctx *ctx;
 
 	ctx = crypto_shash_ctx(tfm);
-	dctx->base = *ctx;
-	dctx->keysize = 0;
-	if (crypto_shash_get_flags(tfm) & CRYPTO_POLY1305_TFM_HAS_KEY)
-		dctx->keysize = POLY1305_KEY_SIZE;
+	dctx->state = ctx->state;
 	return 0;
 }
 

@@ -77,37 +73,10 @@ static inline int crypto_poly1305_update(struct shash_desc *desc,
 {
 	struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
 
-	/*
-	 * The key is passed as the first 32 "data" bytes.  The actual
-	 * poly1305_init() can be called only once the full key is available.
-	 */
-	if (dctx->keysize < POLY1305_KEY_SIZE) {
-		if (!dctx->keysize) {
-			if (arch)
-				poly1305_block_init_arch(&dctx->base.state,
-							 src);
-			else
-				poly1305_block_init_generic(&dctx->base.state,
-							    src);
-		}
-		dctx->keysize += POLY1305_BLOCK_SIZE;
-		src += POLY1305_BLOCK_SIZE;
-		srclen -= POLY1305_BLOCK_SIZE;
-		if (srclen < POLY1305_BLOCK_SIZE)
-			return srclen;
-
-		memcpy(&dctx->base.s, src, POLY1305_BLOCK_SIZE);
-		dctx->keysize += POLY1305_BLOCK_SIZE;
-		src += POLY1305_BLOCK_SIZE;
-		srclen -= POLY1305_BLOCK_SIZE;
-		if (srclen < POLY1305_BLOCK_SIZE)
-			return srclen;
-	}
-
 	if (arch)
-		poly1305_blocks_arch(&dctx->base.state, src, srclen, 1);
+		poly1305_blocks_arch(&dctx->state, src, srclen, 1);
 	else
-		poly1305_blocks_generic(&dctx->base.state, src, srclen, 1);
+		poly1305_blocks_generic(&dctx->state, src, srclen, 1);
 
 	return srclen % POLY1305_BLOCK_SIZE;
 }

@@ -127,29 +96,27 @@ static int crypto_poly1305_update_arch(struct shash_desc *desc,
 static inline int crypto_poly1305_finup(struct shash_desc *desc, const u8 *src,
 					unsigned int len, u8 *dst, bool arch)
 {
+	struct crypto_poly1305_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
 
-	if (unlikely(dctx->keysize != POLY1305_KEY_SIZE))
-		return -ENOKEY;
-
 	if (unlikely(len)) {
 		u8 block[POLY1305_BLOCK_SIZE] = {};
 
 		memcpy(block, src, len);
 		block[len] = 1;
 		if (arch)
-			poly1305_blocks_arch(&dctx->base.state, block,
+			poly1305_blocks_arch(&dctx->state, block,
 					     POLY1305_BLOCK_SIZE, 0);
 		else
-			poly1305_blocks_generic(&dctx->base.state, block,
+			poly1305_blocks_generic(&dctx->state, block,
 						POLY1305_BLOCK_SIZE, 0);
 		memzero_explicit(block, sizeof(block));
 	}
 
 	if (arch)
-		poly1305_emit_arch(&dctx->base.state.h, dst, dctx->base.s);
+		poly1305_emit_arch(&dctx->state.h, dst, ctx->s);
 	else
-		poly1305_emit_generic(&dctx->base.state.h, dst, dctx->base.s);
+		poly1305_emit_generic(&dctx->state.h, dst, ctx->s);
 	return 0;
 }
 

@@ -171,8 +138,7 @@ static struct shash_alg poly1305_algs[] = {
 		.base.cra_name		= "poly1305",
 		.base.cra_driver_name	= "poly1305-generic",
 		.base.cra_priority	= 100,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_ALG_OPTIONAL_KEY,
+		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct crypto_poly1305_ctx),
 		.base.cra_module	= THIS_MODULE,

@@ -187,8 +153,7 @@ static struct shash_alg poly1305_algs[] = {
 		.base.cra_name		= "poly1305",
 		.base.cra_driver_name	= "poly1305-" __stringify(ARCH),
 		.base.cra_priority	= 300,
-		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY |
-					  CRYPTO_ALG_OPTIONAL_KEY,
+		.base.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.base.cra_blocksize	= POLY1305_BLOCK_SIZE,
 		.base.cra_ctxsize	= sizeof(struct crypto_poly1305_ctx),
 		.base.cra_module	= THIS_MODULE,

index 0a3b5718df85..c4e653688ea0 100644
@@ -15,17 +15,15 @@
  * ARMv8 Crypto Extensions instructions to implement the finite field operations.
  */
 
-#include <crypto/algapi.h>
+#include <asm/neon.h>
 #include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
 #include <crypto/polyval.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
+#include <crypto/utils.h>
+#include <linux/cpufeature.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/cpufeature.h>
-#include <asm/neon.h>
-#include <asm/simd.h>
+#include <linux/string.h>
 
 #define NUM_KEY_POWERS	8
 

@@ -38,7 +36,6 @@ struct polyval_tfm_ctx {
 
 struct polyval_desc_ctx {
 	u8 buffer[POLYVAL_BLOCK_SIZE];
-	u32 bytes;
 };
 
 asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys,

@@ -48,25 +45,16 @@ asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2);
 static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
 	const u8 *in, size_t nblocks, u8 *accumulator)
 {
-	if (likely(crypto_simd_usable())) {
-		kernel_neon_begin();
-		pmull_polyval_update(keys, in, nblocks, accumulator);
-		kernel_neon_end();
-	} else {
-		polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
-			nblocks, accumulator);
-	}
+	kernel_neon_begin();
+	pmull_polyval_update(keys, in, nblocks, accumulator);
+	kernel_neon_end();
 }
 
 static void internal_polyval_mul(u8 *op1, const u8 *op2)
 {
-	if (likely(crypto_simd_usable())) {
-		kernel_neon_begin();
-		pmull_polyval_mul(op1, op2);
-		kernel_neon_end();
-	} else {
-		polyval_mul_non4k(op1, op2);
-	}
+	kernel_neon_begin();
+	pmull_polyval_mul(op1, op2);
+	kernel_neon_end();
 }
 
 static int polyval_arm64_setkey(struct crypto_shash *tfm,

@@ -103,49 +91,27 @@ static int polyval_arm64_update(struct shash_desc *desc,
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
 	const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
-	u8 *pos;
 	unsigned int nblocks;
-	unsigned int n;
 
-	if (dctx->bytes) {
-		n = min(srclen, dctx->bytes);
-		pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
-
-		dctx->bytes -= n;
-		srclen -= n;
-
-		while (n--)
-			*pos++ ^= *src++;
-
-		if (!dctx->bytes)
-			internal_polyval_mul(dctx->buffer,
-					    tctx->key_powers[NUM_KEY_POWERS-1]);
-	}
-
-	while (srclen >= POLYVAL_BLOCK_SIZE) {
+	do {
 		/* allow rescheduling every 4K bytes */
 		nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
 		internal_polyval_update(tctx, src, nblocks, dctx->buffer);
 		srclen -= nblocks * POLYVAL_BLOCK_SIZE;
 		src += nblocks * POLYVAL_BLOCK_SIZE;
-	}
+	} while (srclen >= POLYVAL_BLOCK_SIZE);
 
-	if (srclen) {
-		dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
-		pos = dctx->buffer;
-		while (srclen--)
-			*pos++ ^= *src++;
-	}
-
-	return 0;
+	return srclen;
 }
 
-static int polyval_arm64_final(struct shash_desc *desc, u8 *dst)
+static int polyval_arm64_finup(struct shash_desc *desc, const u8 *src,
+			       unsigned int len, u8 *dst)
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
 	const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
 
-	if (dctx->bytes) {
+	if (len) {
+		crypto_xor(dctx->buffer, src, len);
 		internal_polyval_mul(dctx->buffer,
 				     tctx->key_powers[NUM_KEY_POWERS-1]);
 	}

@@ -159,13 +125,14 @@ static struct shash_alg polyval_alg = {
 	.digestsize	= POLYVAL_DIGEST_SIZE,
 	.init		= polyval_arm64_init,
 	.update		= polyval_arm64_update,
-	.final		= polyval_arm64_final,
+	.finup		= polyval_arm64_finup,
 	.setkey		= polyval_arm64_setkey,
 	.descsize	= sizeof(struct polyval_desc_ctx),
 	.base		= {
 		.cra_name		= "polyval",
 		.cra_driver_name	= "polyval-ce",
 		.cra_priority		= 200,
+		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.cra_blocksize		= POLYVAL_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct polyval_tfm_ctx),
 		.cra_module		= THIS_MODULE,

index 8fa58b0f3cb3..6b466867f91a 100644
@@ -16,16 +16,15 @@
  * operations.
  */
 
-#include <crypto/algapi.h>
+#include <asm/cpu_device_id.h>
+#include <asm/fpu/api.h>
 #include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
 #include <crypto/polyval.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
+#include <crypto/utils.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <asm/cpu_device_id.h>
-#include <asm/simd.h>
+#include <linux/string.h>
 
 #define POLYVAL_ALIGN	16
 #define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN)

@@ -42,7 +41,6 @@ struct polyval_tfm_ctx {
 
 struct polyval_desc_ctx {
 	u8 buffer[POLYVAL_BLOCK_SIZE];
-	u32 bytes;
 };
 
 asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,

@@ -57,25 +55,16 @@ static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm)
 static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
 	const u8 *in, size_t nblocks, u8 *accumulator)
 {
-	if (likely(crypto_simd_usable())) {
-		kernel_fpu_begin();
-		clmul_polyval_update(keys, in, nblocks, accumulator);
-		kernel_fpu_end();
-	} else {
-		polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
-			nblocks, accumulator);
-	}
+	kernel_fpu_begin();
+	clmul_polyval_update(keys, in, nblocks, accumulator);
+	kernel_fpu_end();
 }
 
 static void internal_polyval_mul(u8 *op1, const u8 *op2)
 {
-	if (likely(crypto_simd_usable())) {
-		kernel_fpu_begin();
-		clmul_polyval_mul(op1, op2);
-		kernel_fpu_end();
-	} else {
-		polyval_mul_non4k(op1, op2);
-	}
+	kernel_fpu_begin();
+	clmul_polyval_mul(op1, op2);
+	kernel_fpu_end();
 }
 
 static int polyval_x86_setkey(struct crypto_shash *tfm,

@@ -112,49 +101,27 @@ static int polyval_x86_update(struct shash_desc *desc,
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
 	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
-	u8 *pos;
 	unsigned int nblocks;
-	unsigned int n;
 
-	if (dctx->bytes) {
-		n = min(srclen, dctx->bytes);
-		pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
-
-		dctx->bytes -= n;
-		srclen -= n;
-
-		while (n--)
-			*pos++ ^= *src++;
-
-		if (!dctx->bytes)
-			internal_polyval_mul(dctx->buffer,
-					    tctx->key_powers[NUM_KEY_POWERS-1]);
-	}
-
-	while (srclen >= POLYVAL_BLOCK_SIZE) {
+	do {
 		/* Allow rescheduling every 4K bytes. */
 		nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
 		internal_polyval_update(tctx, src, nblocks, dctx->buffer);
 		srclen -= nblocks * POLYVAL_BLOCK_SIZE;
 		src += nblocks * POLYVAL_BLOCK_SIZE;
-	}
+	} while (srclen >= POLYVAL_BLOCK_SIZE);
 
-	if (srclen) {
-		dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
-		pos = dctx->buffer;
-		while (srclen--)
-			*pos++ ^= *src++;
-	}
-
-	return 0;
+	return srclen;
 }
 
-static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
+static int polyval_x86_finup(struct shash_desc *desc, const u8 *src,
+			     unsigned int len, u8 *dst)
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
 	const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm);
 
-	if (dctx->bytes) {
+	if (len) {
+		crypto_xor(dctx->buffer, src, len);
 		internal_polyval_mul(dctx->buffer,
 				     tctx->key_powers[NUM_KEY_POWERS-1]);
 	}

@@ -168,13 +135,14 @@ static struct shash_alg polyval_alg = {
 	.digestsize	= POLYVAL_DIGEST_SIZE,
 	.init		= polyval_x86_init,
 	.update		= polyval_x86_update,
-	.final		= polyval_x86_final,
+	.finup		= polyval_x86_finup,
 	.setkey		= polyval_x86_setkey,
 	.descsize	= sizeof(struct polyval_desc_ctx),
 	.base		= {
 		.cra_name		= "polyval",
 		.cra_driver_name	= "polyval-clmulni",
 		.cra_priority		= 200,
+		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.cra_blocksize		= POLYVAL_BLOCK_SIZE,
 		.cra_ctxsize		= POLYVAL_CTX_SIZE,
 		.cra_module		= THIS_MODULE,

index 4f98910bcdb5..ffd174e75420 100644
@@ -44,15 +44,15 @@
  *
  */
 
-#include <linux/unaligned.h>
-#include <crypto/algapi.h>
 #include <crypto/gf128mul.h>
-#include <crypto/polyval.h>
 #include <crypto/internal/hash.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
+#include <crypto/polyval.h>
+#include <crypto/utils.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
 
 struct polyval_tfm_ctx {
 	struct gf128mul_4k *gf128;

@@ -63,7 +63,6 @@ struct polyval_desc_ctx {
 		u8 buffer[POLYVAL_BLOCK_SIZE];
 		be128 buffer128;
 	};
-	u32 bytes;
 };
 
 static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE],

@@ -76,46 +75,6 @@ static void copy_and_reverse(u8 dst[POLYVAL_BLOCK_SIZE],
 	put_unaligned(swab64(b), (u64 *)&dst[0]);
 }
 
-/*
- * Performs multiplication in the POLYVAL field using the GHASH field as a
- * subroutine.  This function is used as a fallback for hardware accelerated
- * implementations when simd registers are unavailable.
- *
- * Note: This function is not used for polyval-generic, instead we use the 4k
- * lookup table implementation for finite field multiplication.
- */
-void polyval_mul_non4k(u8 *op1, const u8 *op2)
-{
-	be128 a, b;
-
-	// Assume one argument is in Montgomery form and one is not.
-	copy_and_reverse((u8 *)&a, op1);
-	copy_and_reverse((u8 *)&b, op2);
-	gf128mul_x_lle(&a, &a);
-	gf128mul_lle(&a, &b);
-	copy_and_reverse(op1, (u8 *)&a);
-}
-EXPORT_SYMBOL_GPL(polyval_mul_non4k);
-
-/*
- * Perform a POLYVAL update using non4k multiplication.  This function is used
- * as a fallback for hardware accelerated implementations when simd registers
- * are unavailable.
- *
- * Note: This function is not used for polyval-generic, instead we use the 4k
- * lookup table implementation of finite field multiplication.
- */
-void polyval_update_non4k(const u8 *key, const u8 *in,
-			  size_t nblocks, u8 *accumulator)
-{
-	while (nblocks--) {
-		crypto_xor(accumulator, in, POLYVAL_BLOCK_SIZE);
-		polyval_mul_non4k(accumulator, key);
-		in += POLYVAL_BLOCK_SIZE;
-	}
-}
-EXPORT_SYMBOL_GPL(polyval_update_non4k);
-
 static int polyval_setkey(struct crypto_shash *tfm,
 			  const u8 *key, unsigned int keylen)
 {

@@ -154,56 +113,53 @@ static int polyval_update(struct shash_desc *desc,
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
 	const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm);
-	u8 *pos;
 	u8 tmp[POLYVAL_BLOCK_SIZE];
-	int n;
 
-	if (dctx->bytes) {
-		n = min(srclen, dctx->bytes);
-		pos = dctx->buffer + dctx->bytes - 1;
-
-		dctx->bytes -= n;
-		srclen -= n;
-
-		while (n--)
-			*pos-- ^= *src++;
-
-		if (!dctx->bytes)
-			gf128mul_4k_lle(&dctx->buffer128, ctx->gf128);
-	}
-
-	while (srclen >= POLYVAL_BLOCK_SIZE) {
+	do {
 		copy_and_reverse(tmp, src);
 		crypto_xor(dctx->buffer, tmp, POLYVAL_BLOCK_SIZE);
 		gf128mul_4k_lle(&dctx->buffer128, ctx->gf128);
 		src += POLYVAL_BLOCK_SIZE;
 		srclen -= POLYVAL_BLOCK_SIZE;
-	}
+	} while (srclen >= POLYVAL_BLOCK_SIZE);
 
-	if (srclen) {
-		dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
-		pos = dctx->buffer + POLYVAL_BLOCK_SIZE - 1;
-		while (srclen--)
-			*pos-- ^= *src++;
-	}
-
-	return 0;
+	return srclen;
 }
 
-static int polyval_final(struct shash_desc *desc, u8 *dst)
+static int polyval_finup(struct shash_desc *desc, const u8 *src,
+			 unsigned int len, u8 *dst)
 {
 	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
-	const struct polyval_tfm_ctx *ctx = crypto_shash_ctx(desc->tfm);
 
-	if (dctx->bytes)
-		gf128mul_4k_lle(&dctx->buffer128, ctx->gf128);
+	if (len) {
+		u8 tmp[POLYVAL_BLOCK_SIZE] = {};
+
+		memcpy(tmp, src, len);
+		polyval_update(desc, tmp, POLYVAL_BLOCK_SIZE);
+	}
 	copy_and_reverse(dst, dctx->buffer);
 	return 0;
 }
 
-static void polyval_exit_tfm(struct crypto_tfm *tfm)
+static int polyval_export(struct shash_desc *desc, void *out)
 {
-	struct polyval_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+
+	copy_and_reverse(out, dctx->buffer);
+	return 0;
+}
+
+static int polyval_import(struct shash_desc *desc, const void *in)
+{
+	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+
+	copy_and_reverse(dctx->buffer, in);
+	return 0;
+}
+
+static void polyval_exit_tfm(struct crypto_shash *tfm)
+{
+	struct polyval_tfm_ctx *ctx = crypto_shash_ctx(tfm);
 
 	gf128mul_free_4k(ctx->gf128);
 }

@@ -212,17 +168,21 @@ static struct shash_alg polyval_alg = {
 	.digestsize	= POLYVAL_DIGEST_SIZE,
 	.init		= polyval_init,
 	.update		= polyval_update,
-	.final		= polyval_final,
+	.finup		= polyval_finup,
 	.setkey		= polyval_setkey,
+	.export		= polyval_export,
+	.import		= polyval_import,
+	.exit_tfm	= polyval_exit_tfm,
+	.statesize	= sizeof(struct polyval_desc_ctx),
 	.descsize	= sizeof(struct polyval_desc_ctx),
 	.base		= {
 		.cra_name		= "polyval",
 		.cra_driver_name	= "polyval-generic",
 		.cra_priority		= 100,
+		.cra_flags		= CRYPTO_AHASH_ALG_BLOCK_ONLY,
 		.cra_blocksize		= POLYVAL_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct polyval_tfm_ctx),
 		.cra_module		= THIS_MODULE,
-		.cra_exit		= polyval_exit_tfm,
 	},
 };
 

index 1d630f371f77..d2e63743e592 100644
@@ -8,15 +8,7 @@
 #ifndef _CRYPTO_POLYVAL_H
 #define _CRYPTO_POLYVAL_H
 
-#include <linux/types.h>
-#include <linux/crypto.h>
-
 #define POLYVAL_BLOCK_SIZE	16
 #define POLYVAL_DIGEST_SIZE	16
 
-void polyval_mul_non4k(u8 *op1, const u8 *op2);
-
-void polyval_update_non4k(const u8 *key, const u8 *in,
-			  size_t nblocks, u8 *accumulator);
-
 #endif