Skip to contentSkip to conversation

crypto: Add PCLMULQDQ accelerated GHASH implementation

PATCH In review Huang Ying 2 participants 3 messages Sep 15, 2009 – Sep 16, 2009

Subsystems (2)
lore
Share & tools
lore

Huang Ying

author
#0 lore raw

PATCH 1/1 crypto: Add PCLMULQDQ accelerated GHASH implementation

Hi, Herbert,

The dependency to irq_fpu_usable has been merged by linus' tree.

Best Regards,
Huang Ying --------------------------------------------------------------------------> PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:

http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/

Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.

v3:
- Revise GHASH implementation, performance increase about 2x.

---
 arch/x86/crypto/Makefile                   |    3 
 arch/x86/crypto/ghash-clmulni-intel_asm.S  |  157 +++++++++++++
 arch/x86/crypto/ghash-clmulni-intel_glue.c |  331 +++++++++++++++++++++++++++++
 arch/x86/include/asm/cpufeature.h          |    1 
 crypto/Kconfig                             |    8 
 crypto/cryptd.c                            |    7 
 include/crypto/cryptd.h                    |    1 
 7 files changed, 508 insertions(+)
 create mode 100644 arch/x86/crypto/ghash-clmulni-intel_asm.S
 create mode 100644 arch/x86/crypto/ghash-clmulni-intel_glue.c
arch/x86/crypto/Makefile +3 -0
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
 obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
+obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 
 obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
 
@@ -24,3 +25,5 @@ twofish-x86_64-y := twofish-x86_64-asm_6
 salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
 
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
+
+ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
arch/x86/crypto/ghash-clmulni-intel_asm.S new file +157 -0

157 lines not on this page · raw message

arch/x86/crypto/ghash-clmulni-intel_glue.c new file +331 -0

331 lines not on this page · raw message

arch/x86/include/asm/cpufeature.h +1 -0
@@ -247,6 +247,7 @@ extern const char * const x86_power_flag
 #define cpu_has_x2apic		boot_cpu_has(X86_FEATURE_X2APIC)
 #define cpu_has_xsave		boot_cpu_has(X86_FEATURE_XSAVE)
 #define cpu_has_hypervisor	boot_cpu_has(X86_FEATURE_HYPERVISOR)
+#define cpu_has_pclmulqdq	boot_cpu_has(X86_FEATURE_PCLMULQDQ)
 
 #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
 # define cpu_has_invlpg		1
crypto/Kconfig +8 -0
@@ -440,6 +440,14 @@ config CRYPTO_WP512
 	  See also:
 	  <http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html>
 
+config CRYPTO_GHASH_CLMUL_NI_INTEL
+	tristate "GHASH digest algorithm (CLMUL-NI accelerated)"
+	select CRYPTO_SHASH
+	select CRYPTO_CRYPTD
+	help
+	  GHASH is message digest algorithm for GCM (Galois/Counter Mode).
+	  The implementation is accelerated by CLMUL-NI of Intel.
+
 comment "Ciphers"
 
 config CRYPTO_AES
crypto/cryptd.c +7 -0
@@ -711,6 +711,13 @@ struct crypto_shash *cryptd_ahash_child(
 }
 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
 
+struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
+{
+	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
+	return &rctx->desc;
+}
+EXPORT_SYMBOL_GPL(cryptd_shash_desc);
+
 void cryptd_free_ahash(struct cryptd_ahash *tfm)
 {
 	crypto_free_ahash(&tfm->base);
include/crypto/cryptd.h +1 -0
@@ -39,6 +39,7 @@ static inline struct cryptd_ahash *__cry
 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
 					u32 type, u32 mask);
 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm);
+struct shash_desc *cryptd_shash_desc(struct ahash_request *req);
 void cryptd_free_ahash(struct cryptd_ahash *tfm);
 
 #endif
#1 lore raw
Quoted 13 lines · Huang Ying

Hi, Herbert,

The dependency to irq_fpu_usable has been merged by linus' tree.

Best Regards,
Huang Ying --------------------------------------------------------------------------> PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:

http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/

Quoted 7 lines

Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.

v3:
- Revise GHASH implementation, performance increase about 2x.

You have three style issues in this patch, for instance you shouldn't set values inside if() statements .. If you run checkpatch on this it will tell you all the style problems. Could you run this through checkpatch and fix any errors it's find ?

Daniel

author
#2 lore raw
Quoted 19 lines · Daniel Walker
Quoted 18 lines · Huang Ying

Hi, Herbert,

The dependency to irq_fpu_usable has been merged by linus' tree.

Best Regards,
Huang Ying --------------------------------------------------------------------------> PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:

http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/

Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.

Quoted 8 lines
Quoted 2 lines

v3:
- Revise GHASH implementation, performance increase about 2x.

You have three style issues in this patch, for instance you shouldn't set values inside if() statements .. If you run checkpatch on this it will tell you all the style problems. Could you run this through checkpatch and fix any errors it's find ?

Thanks, I will fix them.

Best Regards,
Huang Ying