crypto: riscv: scalar accelerated GHASH

From: Qingfang Deng <[email protected]>

Add a scalar implementation of GHASH for RISC-V using the Zbc (carry-less multiplication) and Zbb (bit-manipulation) extensions. This implementation is adapted from OpenSSL but rewritten in plain C for clarity.

Unlike the OpenSSL one that rely on bit-reflection of the data, this version uses a pre-computed (reflected and multiplied) key, inspired by the approach used in Intel's CLMUL driver, to avoid reflections during runtime.

Signed-off-by Qingfang Deng <[email protected]>

---
 arch/riscv/crypto/Kconfig               |  16 +-
 arch/riscv/crypto/Makefile              |   2 +
 arch/riscv/crypto/ghash-riscv64-clmul.c | 270 ++++++++++++++++++++++++
 3 files changed, 287 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/crypto/ghash-riscv64-clmul.c

arch/riscv/crypto/Kconfig +15 -1

index 6392e1e11bc9..03b74d4116cb 100644
@@ -26,7 +26,7 @@ config CRYPTO_CHACHA_RISCV64
        default CRYPTO_LIB_CHACHA_INTERNAL
 
 config CRYPTO_GHASH_RISCV64
-       tristate "Hash functions: GHASH"
+       tristate "Hash functions: GHASH (vector accelarated)"
        depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
        select CRYPTO_GCM
        help

@@ -35,6 +35,20 @@ config CRYPTO_GHASH_RISCV64
          Architecture: riscv64 using:
          - Zvkg vector crypto extension
 
+config CRYPTO_GHASH_RISCV64_CLMUL
+       tristate "Hash functions: GHASH (CLMUL scalar accelerated)"
+       depends on 64BIT && TOOLCHAIN_HAS_ZBB && TOOLCHAIN_HAS_ZBC
+       select CRYPTO_GCM
+       help
+         GCM GHASH function (NIST SP 800-38D)
+
+         Architecture: riscv64 using:
+         - Zbb Bitmanipulation extension
+         - Zbc Carry-less multiplication
+           OR
+         - Zbkb Bit-manipulation for Cryptography
+         - Zbkc Carry-less multiplication for Cryptography
+
 config CRYPTO_SHA256_RISCV64
        tristate "Hash functions: SHA-224 and SHA-256"
        depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO

arch/riscv/crypto/Makefile +2 -0

index 247c7bc7288c..b5dc497d398c 100644
@@ -10,6 +10,8 @@ chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
 obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
 ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
 
+obj-$(CONFIG_CRYPTO_GHASH_RISCV64_CLMUL) += ghash-riscv64-clmul.o
+
 obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
 sha256-riscv64-y := sha256-riscv64-glue.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o
 

arch/riscv/crypto/ghash-riscv64-clmul.c new file +34 -0

new file mode 100644
index 000000000000..4777aa8e94cb
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GHASH using the RISC-V Zbc/Zbkc (CLMUL) extension
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Christoph Müllner <[email protected]>
+ *
+ * Copyright (C) 2025 Siflower Communications Ltd
+ * Author: Qingfang Deng <[email protected]>
+ */
+
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <crypto/ghash.h>
+#include <crypto/internal/hash.h>
+
+#define GHASH_MOD_POLY 0xc200000000000000
+
+struct riscv64_clmul_ghash_ctx {
+       __uint128_t key;
+};
+
+struct riscv64_clmul_ghash_desc_ctx {
+       __uint128_t shash;
+       u8 buffer[GHASH_DIGEST_SIZE];
+       int bytes;
+};
+
+static __always_inline u64 riscv_zbb_swab64(u64 val)
+{
+       asm (".option push\n"
+            ".option arch,+zbb\n"

arch/riscv/crypto/ghash-riscv64-clmul.c new file +226 -1

+       val = *(__uint128_t *)p;
+       val = riscv_zbb_swab64(val >> 64) | (__uint128_t)riscv_zbb_swab64(val) << 64;
+#else
+       val = (__uint128_t)p[0] << 120;
+       val |= (__uint128_t)p[1] << 112;
+       val |= (__uint128_t)p[2] << 104;
+       val |= (__uint128_t)p[3] << 96;
+       val |= (__uint128_t)p[4] << 88;
+       val |= (__uint128_t)p[5] << 80;
+       val |= (__uint128_t)p[6] << 72;
+       val |= (__uint128_t)p[7] << 64;
+       val |= (__uint128_t)p[8] << 56;
+       val |= (__uint128_t)p[9] << 48;
+       val |= (__uint128_t)p[10] << 40;
+       val |= (__uint128_t)p[11] << 32;
+       val |= (__uint128_t)p[12] << 24;
+       val |= (__uint128_t)p[13] << 16;
+       val |= (__uint128_t)p[14] << 8;
+       val |= (__uint128_t)p[15];
+#endif
+       return val;
+}
+
+static __always_inline void put_unaligned_be128(__uint128_t val, u8 *p)
+{
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+       *(__uint128_t *)p = riscv_zbb_swab64(val >> 64) | (__uint128_t)riscv_zbb_swab64(val) << 64;
+#else
+       p[0] = val >> 120;
+       p[1] = val >> 112;
+       p[2] = val >> 104;
+       p[3] = val >> 96;
+       p[4] = val >> 88;
+       p[5] = val >> 80;
+       p[6] = val >> 72;
+       p[7] = val >> 64;
+       p[8] = val >> 56;
+       p[9] = val >> 48;
+       p[10] = val >> 40;
+       p[11] = val >> 32;
+       p[12] = val >> 24;
+       p[13] = val >> 16;
+       p[14] = val >> 8;
+       p[15] = val;
+#endif
+}
+
+static __always_inline __attribute_const__
+__uint128_t clmul128(u64 a, u64 b)
+{
+       u64 hi, lo;
+
+       asm(".option push\n"
+           ".option arch,+zbc\n"
+           "clmul      %0, %2, %3\n"
+           "clmulh     %1, %2, %3\n"
+           ".option pop\n"
+           : "=&r" (lo), "=&r" (hi) : "r" (a), "r" (b));
+       return (__uint128_t)hi << 64 | lo;
+}
+
+static int riscv64_clmul_ghash_init(struct shash_desc *desc)
+{
+       struct riscv64_clmul_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
+
+       dctx->bytes = 0;
+       dctx->shash = 0;
+       return 0;
+}
+
+/* Compute GMULT (Xi*H mod f) using the Zbc (clmul) extensions.
+ * Using the no-Karatsuba approach and clmul for the final reduction.
+ * This results in an implementation with minimized number of instructions.
+ * HW with clmul latencies higher than 2 cycles might observe a performance
+ * improvement with Karatsuba. HW with clmul latencies higher than 6 cycles
+ * might observe a performance improvement with additionally converting the
+ * reduction to shift&xor. For a full discussion of this estimates see
+ * https://github.com/riscv/riscv-crypto/blob/master/doc/supp/gcm-mode-cmul.adoc
+ */
+static void gcm_ghash_rv64i_zbc(__uint128_t *Xi, __uint128_t k, const u8 *inp, size_t len)
+{
+       u64 k_hi = k >> 64, k_lo = k, p_hi, p_lo;
+       __uint128_t hash = *Xi, p;
+
+       do {
+               __uint128_t t0, t1, t2, t3, lo, mid, hi;
+
+               /* Load the input data, byte-reverse them, and XOR them with Xi */
+               p = get_unaligned_be128(inp);
+
+               inp += GHASH_BLOCK_SIZE;
+               len -= GHASH_BLOCK_SIZE;
+
+               p ^= hash;
+               p_hi = p >> 64;
+               p_lo = p;
+
+               /* Multiplication (without Karatsuba) */
+               t0 = clmul128(p_lo, k_lo);
+               t1 = clmul128(p_lo, k_hi);
+               t2 = clmul128(p_hi, k_lo);
+               t3 = clmul128(p_hi, k_hi);
+               mid = t1 ^ t2;
+               lo = t0 ^ (mid << 64);
+               hi = t3 ^ (mid >> 64);
+
+               /* Reduction with clmul */
+               mid = clmul128(lo, GHASH_MOD_POLY);
+               lo ^= mid << 64;
+               hi ^= lo ^ (mid >> 64);
+               hi ^= clmul128(lo >> 64, GHASH_MOD_POLY);
+               hash = hi;
+       } while (len);
+
+       *Xi = hash;
+}
+
+static int riscv64_clmul_ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen)
+{
+       struct riscv64_clmul_ghash_ctx *ctx = crypto_shash_ctx(tfm);
+       __uint128_t k;
+
+       if (keylen != GHASH_BLOCK_SIZE)
+               return -EINVAL;
+
+       k = get_unaligned_be128(key);
+       k = (k << 1 | k >> 127) ^ (k >> 127 ? (__uint128_t)GHASH_MOD_POLY << 64 : 0);
+       ctx->key = k;
+
+       return 0;
+}
+
+static int riscv64_clmul_ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen)
+{
+       struct riscv64_clmul_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
+       struct riscv64_clmul_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
+       unsigned int len;
+
+       if (dctx->bytes) {
+               if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
+                       memcpy(dctx->buffer + dctx->bytes, src, srclen);
+                       dctx->bytes += srclen;
+                       return 0;
+               }
+               memcpy(dctx->buffer + dctx->bytes, src, GHASH_DIGEST_SIZE - dctx->bytes);
+
+               gcm_ghash_rv64i_zbc(&dctx->shash, ctx->key, dctx->buffer, GHASH_DIGEST_SIZE);
+
+               src += GHASH_DIGEST_SIZE - dctx->bytes;
+               srclen -= GHASH_DIGEST_SIZE - dctx->bytes;
+               dctx->bytes = 0;
+       }
+
+       len = round_down(srclen, GHASH_BLOCK_SIZE);
+       if (len) {
+               gcm_ghash_rv64i_zbc(&dctx->shash, ctx->key, src, len);
+               src += len;
+               srclen -= len;
+       }
+
+       if (srclen) {
+               memcpy(dctx->buffer, src, srclen);
+               dctx->bytes = srclen;
+       }
+       return 0;
+}
+
+static int riscv64_clmul_ghash_final(struct shash_desc *desc, u8 out[GHASH_DIGEST_SIZE])
+{
+       struct riscv64_clmul_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
+       struct riscv64_clmul_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
+       int i;
+
+       if (dctx->bytes) {
+               for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++)
+                       dctx->buffer[i] = 0;
+               gcm_ghash_rv64i_zbc(&dctx->shash, ctx->key, dctx->buffer, GHASH_DIGEST_SIZE);
+               dctx->bytes = 0;
+       }
+       put_unaligned_be128(dctx->shash, out);
+       return 0;
+}
+
+struct shash_alg riscv64_clmul_ghash_alg = {
+       .init = riscv64_clmul_ghash_init,
+       .update = riscv64_clmul_ghash_update,
+       .final = riscv64_clmul_ghash_final,
+       .setkey = riscv64_clmul_ghash_setkey,
+       .descsize = sizeof(struct riscv64_clmul_ghash_desc_ctx),
+       .digestsize = GHASH_DIGEST_SIZE,
+       .base = {
+                .cra_blocksize = GHASH_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct riscv64_clmul_ghash_ctx),
+                .cra_priority = 250,
+                .cra_name = "ghash",
+                .cra_driver_name = "ghash-riscv64-clmul",
+                .cra_module = THIS_MODULE,
+       },
+};
+
+static int __init riscv64_clmul_ghash_mod_init(void)
+{
+       bool has_clmul, has_rev8;
+
+       has_clmul = riscv_isa_extension_available(NULL, ZBC) ||
+                   riscv_isa_extension_available(NULL, ZBKC);
+       has_rev8 = riscv_isa_extension_available(NULL, ZBB) ||
+                  riscv_isa_extension_available(NULL, ZBKB);
+       if (has_clmul && has_rev8)
+               return crypto_register_shash(&riscv64_clmul_ghash_alg);
+
+       return -ENODEV;
+}
+
+static void __exit riscv64_clmul_ghash_mod_fini(void)
+{
+       crypto_unregister_shash(&riscv64_clmul_ghash_alg);
+}
+
+module_init(riscv64_clmul_ghash_mod_init);
+module_exit(riscv64_clmul_ghash_mod_fini);
+
+MODULE_DESCRIPTION("GHASH (RISC-V CLMUL accelerated)");
+MODULE_AUTHOR("Qingfang Deng <[email protected]>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_CRYPTO("ghash");
--

2.43.0

(cc Eric)

Quoted 12 lines · Qingfang Deng

From: Qingfang Deng <[email protected]>

Add a scalar implementation of GHASH for RISC-V using the Zbc (carry-less multiplication) and Zbb (bit-manipulation) extensions. This implementation is adapted from OpenSSL but rewritten in plain C for clarity.

Unlike the OpenSSL one that rely on bit-reflection of the data, this version uses a pre-computed (reflected and multiplied) key, inspired by the approach used in Intel's CLMUL driver, to avoid reflections during runtime.

Quoted 1 line

Signed-off-by Qingfang Deng <[email protected]>

What is the use case for this? AIUI, the scalar AES instructions were never implemented by anyone, so how do you expect this to be used in practice?

...

arch/riscv/crypto/ghash-riscv64-clmul.c line 44

Quoted 4 lines

arch/riscv/crypto/ghash-riscv64-clmul.c new file +4 -0

+static __always_inline __uint128_t get_unaligned_be128(const u8 *p)
+{
+       __uint128_t val;
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS

CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS means that get_unaligned_xxx() helpers are cheap. Casting a void* to an aligned type is still UB as per the C standard.

So better to drop the #ifdef entirely, and just use the get_unaligned_be64() helpers for both cases.

(same below)

Also, do you need to test for int128 support? Or is that guaranteed for all compilers that are supported by the RISC-V port?

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

[...]

Also, do you need to test for int128 support? Or is that guaranteed for all compilers that are supported by the RISC-V port?

I believe int128 support is available for all 64-bit targets.

Quoted 10 lines · Qingfang Deng

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

[...]

Also, do you need to test for int128 support? Or is that guaranteed for all compilers that are supported by the RISC-V port?

I believe int128 support is available for all 64-bit targets.

You can verify the compiler supports int128 with the following macro:

    #if (__SIZEOF_INT128__ >= 16)
    ...
    #endif

Also see <https://gcc.gnu.org/pipermail/gcc-help/2015-August/124862.html>.

Jeff

Hi Ard,

Quoted 16 lines · Ard Biesheuvel

(cc Eric)

Quoted 12 lines · Qingfang Deng

From: Qingfang Deng <[email protected]>

Add a scalar implementation of GHASH for RISC-V using the Zbc (carry-less multiplication) and Zbb (bit-manipulation) extensions. This implementation is adapted from OpenSSL but rewritten in plain C for clarity.

Unlike the OpenSSL one that rely on bit-reflection of the data, this version uses a pre-computed (reflected and multiplied) key, inspired by the approach used in Intel's CLMUL driver, to avoid reflections during runtime.

Quoted 5 lines

Quoted 1 line

Signed-off-by Qingfang Deng <[email protected]>

What is the use case for this? AIUI, the scalar AES instructions were never implemented by anyone, so how do you expect this to be used in practice?

The use case _is_ AES-GCM, as you mentioned. Without this, computing GHASH can take a considerable amount of CPU time (monitored by perf).

Quoted 9 lines

...

arch/riscv/crypto/ghash-riscv64-clmul.c line 44

Quoted 4 lines

arch/riscv/crypto/ghash-riscv64-clmul.c new file +4 -0

+static __always_inline __uint128_t get_unaligned_be128(const u8 *p)
+{
+       __uint128_t val;
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS

CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS means that get_unaligned_xxx() helpers are cheap. Casting a void* to an aligned type is still UB as per the C standard.

Technically an unaligned access is UB but this pattern is widely used in networking code.

Quoted 3 lines

So better to drop the #ifdef entirely, and just use the get_unaligned_be64() helpers for both cases.

Currently those helpers won't generate rev8 instructions, even if HAVE_EFFICIENT_UNALIGNED_ACCESS and RISCV_ISA_ZBB is set, so I have to implement my own version of this to reduce the number of instructions, and to align with the original OpenSSL implementation.

Quoted 5 lines

Quoted 4 lines

What is the use case for this? AIUI, the scalar AES instructions were never implemented by anyone, so how do you expect this to be used in practice?

Quoted 3 lines

The use case _is_ AES-GCM, as you mentioned. Without this, computing GHASH can take a considerable amount of CPU time (monitored by perf).

I see. But do you have a particular configuration in mind? Does it have scalar AES too? I looked into that a while ago but I was told that nobody actually incorporates that. So what about these extensions? Are they commonly implemented?

[0] https://web.git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git/log/?h=riscv-scalar-aes

Quoted 10 lines

Quoted 9 lines

...

arch/riscv/crypto/ghash-riscv64-clmul.c line 44

Quoted 4 lines

arch/riscv/crypto/ghash-riscv64-clmul.c new file +4 -0

+static __always_inline __uint128_t get_unaligned_be128(const u8 *p)
+{
+       __uint128_t val;
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS

CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS means that get_unaligned_xxx() helpers are cheap. Casting a void* to an aligned type is still UB as per the C standard.

Quoted 3 lines

Technically an unaligned access is UB but this pattern is widely used in networking code.

Of course. But that is no reason to keep doing it.

Quoted 4 lines

Quoted 3 lines

So better to drop the #ifdef entirely, and just use the get_unaligned_be64() helpers for both cases.

Quoted 5 lines

Currently those helpers won't generate rev8 instructions, even if HAVE_EFFICIENT_UNALIGNED_ACCESS and RISCV_ISA_ZBB is set, so I have to implement my own version of this to reduce the number of instructions, and to align with the original OpenSSL implementation.

So fix the helpers.

Quoted 5 lines · Ard Biesheuvel

Quoted 5 lines

Quoted 4 lines

What is the use case for this? AIUI, the scalar AES instructions were never implemented by anyone, so how do you expect this to be used in practice?

Quoted 8 lines

Quoted 3 lines

The use case _is_ AES-GCM, as you mentioned. Without this, computing GHASH can take a considerable amount of CPU time (monitored by perf).

I see. But do you have a particular configuration in mind? Does it have scalar AES too? I looked into that a while ago but I was told that nobody actually incorporates that. So what about these extensions? Are they commonly implemented?

It's aes-generic.c (LUT-based) with accelerated GHASH.

Quoted 29 lines

[0] https://web.git.kernel.org/pub/scm/linux/kernel/git/ardb/linux.git/log/?h=riscv-scalar-aes

Quoted 10 lines

Quoted 9 lines

...

arch/riscv/crypto/ghash-riscv64-clmul.c line 44

Quoted 4 lines

arch/riscv/crypto/ghash-riscv64-clmul.c new file +4 -0

+static __always_inline __uint128_t get_unaligned_be128(const u8 *p)
+{
+       __uint128_t val;
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS

CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS means that get_unaligned_xxx() helpers are cheap. Casting a void* to an aligned type is still UB as per the C standard.

Quoted 3 lines

Technically an unaligned access is UB but this pattern is widely used in networking code.

Of course. But that is no reason to keep doing it.

Quoted 9 lines

Quoted 3 lines

So better to drop the #ifdef entirely, and just use the get_unaligned_be64() helpers for both cases.

Currently those helpers won't generate rev8 instructions, even if HAVE_EFFICIENT_UNALIGNED_ACCESS and RISCV_ISA_ZBB is set, so I have to implement my own version of this to reduce the number of instructions, and to align with the original OpenSSL implementation.

Quoted 1 line

So fix the helpers.

The issue is that RISC-V GCC doesn’t emit efficient unaligned loads by default:
- Not all RISC-V CPUs support unaligned access efficiently, so GCC falls back to conservative byte-wise code.

- There’s no clean way to force the optimized path - GCC only emits fast unaligned loads if tuned for a specific CPU (e.g., -mtune=size or -mtune=thead-c906), which the kernel doesn't typically do, even with HAVE_EFFICIENT_UNALIGNED_ACCESS.

Maybe we should raise this with the GCC maintainers. An explicit option to enable optimized unaligned access could help.

Quoted 8 lines

- There’s no clean way to force the optimized path - GCC only emits fast unaligned loads if tuned for a specific CPU (e.g., -mtune=size or -mtune=thead-c906), which the kernel doesn't typically do, even with HAVE_EFFICIENT_UNALIGNED_ACCESS.

Maybe we should raise this with the GCC maintainers. An explicit option to enable optimized unaligned access could help.

HAVE_EFFICIENT_UNALIGNED_ACCESS is a build time setting, so the resulting kernel only runs correctly on hardware that implements unaligned accesses in hardware.

So that means you could pass this -mtune= option too in that case, no?

Then, you can just use a packed struct or an __aligned(1) annotation and the compiler will emit the correct code for you, depending on whether unaligned accesses are permitted.

Quoted 4 lines

I see. But do you have a particular configuration in mind? Does it have scalar AES too? I looked into that a while ago but I was told that nobody actually incorporates that. So what about these extensions? Are they commonly implemented?

It's aes-generic.c (LUT-based) with accelerated GHASH.

Quoted 5 lines · Qingfang Deng

Quoted 4 lines

I see. But do you have a particular configuration in mind? Does it have scalar AES too? I looked into that a while ago but I was told that nobody actually incorporates that. So what about these extensions? Are they commonly implemented?

Quoted 1 line

It's aes-generic.c (LUT-based) with accelerated GHASH.

That's an odd combination. Normally accelerated AES and GHASH go together. Presumably you're targeting some specific RISC-V CPU. Is this going to be a one-off thing for that specific CPU? Or can we expect many RISC-V CPUs to have GHASH acceleration without AES? And if so, why?

- Eric

arch/riscv/crypto/ghash-riscv64-clmul.c new file +10 -0

+static int riscv64_clmul_ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen)
+{
+	struct riscv64_clmul_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
+	struct riscv64_clmul_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
+	unsigned int len;
+
+	if (dctx->bytes) {
+		if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
+			memcpy(dctx->buffer + dctx->bytes, src, srclen);
+			dctx->bytes += srclen;

arch/riscv/crypto/ghash-riscv64-clmul.c new file +50 -0

+static __always_inline u64 riscv_zbb_swab64(u64 val)
+{
+	asm (".option push\n"
+	     ".option arch,+zbb\n"
+	     "rev8 %0, %1\n"
+	     ".option pop\n"
+	     : "=r" (val) : "r" (val));
+	return val;
+}
+
+static __always_inline __uint128_t get_unaligned_be128(const u8 *p)
+{
+	__uint128_t val;
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	val = *(__uint128_t *)p;
+	val = riscv_zbb_swab64(val >> 64) | (__uint128_t)riscv_zbb_swab64(val) << 64;
+#else
+	val = (__uint128_t)p[0] << 120;
+	val |= (__uint128_t)p[1] << 112;
+	val |= (__uint128_t)p[2] << 104;
+	val |= (__uint128_t)p[3] << 96;
+	val |= (__uint128_t)p[4] << 88;
+	val |= (__uint128_t)p[5] << 80;
+	val |= (__uint128_t)p[6] << 72;
+	val |= (__uint128_t)p[7] << 64;
+	val |= (__uint128_t)p[8] << 56;
+	val |= (__uint128_t)p[9] << 48;
+	val |= (__uint128_t)p[10] << 40;
+	val |= (__uint128_t)p[11] << 32;
+	val |= (__uint128_t)p[12] << 24;
+	val |= (__uint128_t)p[13] << 16;
+	val |= (__uint128_t)p[14] << 8;
+	val |= (__uint128_t)p[15];
+#endif
+	return val;
+}
+
+static __always_inline void put_unaligned_be128(__uint128_t val, u8 *p)
+{
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	*(__uint128_t *)p = riscv_zbb_swab64(val >> 64) | (__uint128_t)riscv_zbb_swab64(val) << 64;
+#else
+	p[0] = val >> 120;
+	p[1] = val >> 112;
+	p[2] = val >> 104;
+	p[3] = val >> 96;
+	p[4] = val >> 88;
+	p[5] = val >> 80;
+	p[6] = val >> 72;
+	p[7] = val >> 64;

Please help properly optimize swab*() and {get,put}_unaligned_* for RISC-V first, before considering random hacks like this.

https://lore.kernel.org/r/[email protected]
is working on swab*().

arch/riscv/crypto/ghash-riscv64-clmul.c line 149

Quoted 9 lines

arch/riscv/crypto/ghash-riscv64-clmul.c new file +8 -0

+             /* Multiplication (without Karatsuba) */
+             t0 = clmul128(p_lo, k_lo);
+             t1 = clmul128(p_lo, k_hi);
+             t2 = clmul128(p_hi, k_lo);
+             t3 = clmul128(p_hi, k_hi);
+             mid = t1 ^ t2;
+             lo = t0 ^ (mid << 64);
+             hi = t3 ^ (mid >> 64);

There is no need to explicitly XOR 'mid << 64' into lo and 'mid >> 64' into hi. Take a look at how arch/x86/crypto/aes-gcm-*.S do it.

Also, since this is only doing one block at a time and does not use Karatsuba multiplication, the single-step reduction would work well here. See aes-gcm-aesni-x86_64.S.

- Eric