Using Intel CRC32 instruction to accelerate CRC32c algorithm by new crypto API -V3.
Subsystems (2)
PATCH 1/1 Using Intel CRC32 instruction to accelerate CRC32c algorithm by new crypto API -V3.
Revised by comments:
Add 'static' for limitation namespace;
Resend for fixing lines-folded by adjusting evolution config; (The patch was created against 2.6.27-rc1)
Quoted 1 line
From NHM processor onward, Intel processors can support hardware accelerated
CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum.
Byte code instructions are used for compiler compatibility.
No MMX / XMM registers is involved in the implementation.
--- arch/x86/crypto/Makefile | 2 arch/x86/crypto/crc32c-intel.c | 190 +++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 12 ++ include/asm-x86/cpufeature.h | 2 4 files changed, 206 insertions(+)
arch/x86/crypto/crc32c-intel.c +190 -0
@@ -0,0 +1,190 @@ +/* +* Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. +* CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) +* CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: +* http://www.intel.com/products/processor/manuals/ +* Intel(R) 64 and IA-32 Architectures Software Developer's Manual +* Volume 2A: Instruction Set Reference, A-M +*/ +#include <linux/init.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/kernel.h> +#include <crypto/internal/hash.h> + +#include <asm/cpufeature.h> + +#define CHKSUM_BLOCK_SIZE 1 +#define CHKSUM_DIGEST_SIZE 4 + +#ifdef CONFIG_X86_64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif + +static u32 crc32c_intel_le_hw_byte (u32 crc,unsigned char const *data,size_t length) +{ +while (length--) { + __asm__ __volatile__( +".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" + :"=S" (crc) + :"0" (crc),"c" (*data) + ); + data++; + } + +return crc; +} + +static u32 __pure crc32c_intel_le_hw (u32 crc,unsigned char const *p,size_t len) +{ +unsigned int iquotient = len /SCALE_F ; +unsigned int iremainder = len %SCALE_F ; +unsigned long *ptmp = (unsigned long *)p; + +while (iquotient--) { + __asm__ __volatile__( +".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" + :"=S" (crc) + :"0" (crc),"c" (*ptmp) + ); + ptmp++; + } + +if (iremainder) + crc =crc32c_intel_le_hw_byte (crc, (unsigned char *)ptmp, + iremainder); + +return crc; +} + +/* +* Setting the seed allows arbitrary accumulators and flexible XOR policy +* If your algorithm starts with ~0, then XOR with ~0 before you set +* the seed. +*/ +static int crc32c_intel_setkey (struct crypto_ahash *hash,const u8 *key, +unsigned int keylen) +{ +u32 *mctx =crypto_ahash_ctx (hash); + +if (keylen !=sizeof (u32)) { +crypto_ahash_set_flags (hash,CRYPTO_TFM_RES_BAD_KEY_LEN ); +return -EINVAL ; + } + *mctx =le32_to_cpup ((__le32 *)key); +return 0 ; +} + +static int crc32c_intel_init (struct ahash_request *req) +{ +u32 *mctx =crypto_ahash_ctx (crypto_ahash_reqtfm (req)); +u32 *crcp =ahash_request_ctx (req); + + *crcp = *mctx; + +return 0 ; +} + +static int crc32c_intel_update (struct ahash_request *req) +{ +struct crypto_hash_walk walk; +u32 *crcp =ahash_request_ctx (req); +u32 crc = *crcp; +int nbytes; + +for (nbytes =crypto_hash_walk_first (req, &walk); nbytes; + nbytes =crypto_hash_walk_done (&walk,0 )) + crc =crc32c_intel_le_hw (crc, walk.data, nbytes); + + *crcp = crc; +return 0 ; +} + +static int crc32c_intel_final (struct ahash_request *req) +{ +u32 *crcp =ahash_request_ctx (req); + + *(__le32 *)req->result = ~cpu_to_le32p (crcp); +return 0 ; +} + +static int crc32c_intel_digest (struct ahash_request *req) +{ +struct crypto_hash_walk walk; +u32 *mctx =crypto_ahash_ctx (crypto_ahash_reqtfm (req)); +u32 crc = *mctx; +int nbytes; + +for (nbytes =crypto_hash_walk_first (req, &walk); nbytes; + nbytes =crypto_hash_walk_done (&walk,0 )) + crc =crc32c_intel_le_hw (crc, walk.data, nbytes); + + *(__le32 *)req->result = ~cpu_to_le32 (crc); +return 0 ; +} + +static int crc32c_intel_cra_init (struct crypto_tfm *tfm) +{ +u32 *key =crypto_tfm_ctx (tfm); + + *key = ~0 ; + + tfm->crt_ahash.reqsize =sizeof (u32); + +return 0 ; +} + +static struct crypto_alg alg = { + .cra_name ="crc32c" , + .cra_driver_name ="crc32c-intel" , + .cra_priority =200 , + .cra_flags =CRYPTO_ALG_TYPE_AHASH , + .cra_blocksize =CHKSUM_BLOCK_SIZE , + .cra_alignmask =3 , + .cra_ctxsize =sizeof (u32), + .cra_module =THIS_MODULE , + .cra_list =LIST_HEAD_INIT (alg.cra_list), + .cra_init = crc32c_intel_cra_init, + .cra_type = &crypto_ahash_type, + .cra_u = { + .ahash = { + .digestsize =CHKSUM_DIGEST_SIZE , + .setkey = crc32c_intel_setkey, + .init = crc32c_intel_init, + .update = crc32c_intel_update, + .final = crc32c_intel_final, + .digest = crc32c_intel_digest, + } + } +}; + + +static int __init crc32c_intel_mod_init (void ) +{ +if (cpu_has_xmm4_2) +return crypto_register_alg (&alg); +else { +printk (KERN_ERR "No support in current hardware.\n" ); +return -1 ; + } +} + +static void __exit crc32c_intel_mod_fini(void) +{ +crypto_unregister_alg (&alg); +} + +module_init (crc32c_intel_mod_init); +module_exit (crc32c_intel_mod_fini); + +MODULE_AUTHOR ("Austin Zhang <[email protected]>, Kent Liu <[email protected]>" ); +MODULE_DESCRIPTION ("CRC32c (Castagnoli) optimization using Intel Hardware." ); +MODULE_LICENSE ("GPL" ); + +MODULE_ALIAS ("crc32c" ); +MODULE_ALIAS ("crc32c-intel" ); +
arch/x86/crypto/Makefile +2 -0
@@ -10,6 +10,8 @@ obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o +obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o + aes-i586-y := aes-i586-asm_32.o aes_glue.o twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
crypto/Kconfig +12 -0
@@ -221,6 +221,18 @@ config CRYPTO_CRC32C See Castagnoli93. This implementation uses lib/libcrc32c. Module will be crc32c. +config CRYPTO_CRC32C_INTEL + tristate "CRC32c INTEL hardware acceleration" + depends on X86 + select CRYPTO_ALGAPI + help + In Intel processor with SSE4.2 supported, the processor will + support CRC32C implementation using hardware accelerated CRC32 + instruction. This option will create 'crc32c-intel' module, + which will enable any routine to use the CRC32 instruction to + gain performance compared with software implementation. + Module will be crc32c-intel. + config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_ALGAPI
include/asm-x86/cpufeature.h +2 -0
@@ -91,6 +91,7 @@#define X86_FEATURE_CX16 (4*32+13)/* CMPXCHG16B */ #define X86_FEATURE_XTPR (4*32+14)/* Send Task Priority Messages */ #define X86_FEATURE_DCA (4*32+18)/* Direct Cache Access */ +#define X86_FEATURE_XMM4_2 (4*32+20)/* Streaming SIMD Extensions-4.2 */ /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ #define X86_FEATURE_XSTORE (5*32+ 2)/* on-CPU RNG present (xstore insn) */
@@ -189,6 +190,7 @@ extern const char * const x86_power_flag#define cpu_has_gbpages boot_cpu_has(X86_FEATURE_GBPAGES)#define cpu_has_arch_perfmon boot_cpu_has(X86_FEATURE_ARCH_PERFMON)#define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT) +#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2)#if defined(CONFIG_X86_INVLPG ) || defined(CONFIG_X86_64 )# define cpu_has_invlpg 1
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Hi!
Quoted 25 lines
??????Revised by comments:
Add 'static' for limitation namespace;
Resend for fixing lines-folded by adjusting evolution config; (The patch was created against 2.6.27-rc1)Quoted 1 line
From NHM processor onward, Intel processors can support hardware accelerated
CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum.
Byte code instructions are used for compiler compatibility.
No MMX / XMM registers is involved in the implementation.Signed-off-by Austin Zhang <[email protected]>Signed-off-by Kent Liu <[email protected]>--- arch/x86/crypto/Makefile | 2 arch/x86/crypto/crc32c-intel.c | 190 +++++++++++++++++++++++++++++++++++++++++ crypto/Kconfig | 12 ++ include/asm-x86/cpufeature.h | 2 4 files changed, 206 insertions(+)diff -Naurp
linux-2.6/arch/x86/crypto/crc32c-intel.clinux-2.6-patch/arch/x86/crypto/crc32c-intel.c
arch/x86/crypto/crc32c-intel.c line 7
Quoted 7 lines
arch/x86/crypto/crc32c-intel.c +7
-0@@ -0,0 +1,190 @@ +/* +* Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. +* CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) +* CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: +* http://www.intel.com/products/processor/manuals/ +* Intel(R) 64 and IA-32 Architectures Software Developer's Manual +* Volume 2A: Instruction Set Reference, A-M
Copyright / GPL?
arch/x86/crypto/crc32c-intel.c line 26
Quoted 7 lines
arch/x86/crypto/crc32c-intel.c +7
-0+#ifdef CONFIG_X86_64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif
Ouch...
arch/x86/crypto/crc32c-intel.c line 31
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c +4
-0+static u32 crc32c_intel_le_hw_byte (u32 crc,unsigned char const *data,size_t length) +{ +while (length--) { + __asm__ __volatile__(
Are all the underscores neccessary?
arch/x86/crypto/crc32c-intel.c line 32
Quoted 1 line
arch/x86/crypto/crc32c-intel.c +1
-0+".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
Ouch...
arch/x86/crypto/crc32c-intel.c line 112
Quoted 6 lines
arch/x86/crypto/crc32c-intel.c +6
-0+static int crc32c_intel_final (struct ahash_request *req) +{ +u32 *crcp =ahash_request_ctx (req); + + *(__le32 *)req->result = ~cpu_to_le32p (crcp); +return 0 ;
This is not user visible, so le32 should be enough.
arch/x86/crypto/crc32c-intel.c line 171
Quoted 6 lines
arch/x86/crypto/crc32c-intel.c +6
-0+static int __init crc32c_intel_mod_init (void ) +{ +if (cpu_has_xmm4_2) +return crypto_register_alg (&alg); +else { +printk (KERN_ERR "No support in current hardware.\n" );
Missing space.
arch/x86/crypto/crc32c-intel.c line 172
Quoted 1 line
arch/x86/crypto/crc32c-intel.c +1
-0+return -1 ;
That's supposed to be errno, right?
arch/x86/crypto/crc32c-intel.c line 40
Quoted 8 lines
arch/x86/crypto/crc32c-intel.c +8
-0+config CRYPTO_CRC32C_INTEL + tristate"CRC32c INTEL hardware acceleration" + depends onX86 + selectCRYPTO_ALGAPI + help + In Intel processor with SSE4.2 supported, the processor will + support CRC32C implementation using hardware accelerated CRC32 + instruction. This option will create 'crc32c-intel' module,
In Intel processor with SSE4.2, we can use hardware CRC32 acceleration.
arch/x86/crypto/crc32c-intel.c line 43
Quoted 3 lines
arch/x86/crypto/crc32c-intel.c +3
-0+ which will enable any routine to use the CRC32 instruction to + gain performance compared with software implementation. + Module will be crc32c-intel.
crc32c-intel.ko?
Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Herbert Xu
maintainerQuoted 9 lines · Pavel Machek
arch/x86/crypto/crc32c-intel.c line 112
Quoted 7 lines
arch/x86/crypto/crc32c-intel.c +6
-0+static int crc32c_intel_final (struct ahash_request *req) +{ +u32 *crcp =ahash_request_ctx (req); + + *(__le32 *)req->result = ~cpu_to_le32p (crcp); +return 0 ;This is not user visible, so le32 should be enough.
Unfortunately I don't think le32 exists, it'd definitely be nice to have it though.
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtPaval,
Thanks for your comments.
Quoted 1 line · Pavel Machek
Copyright / GPL?
Yes, as : +MODULE_LICENSE("GPL");
Quoted 9 lines
arch/x86/crypto/crc32c-intel.c line 26
Quoted 7 lines
arch/x86/crypto/crc32c-intel.c +7
-0+#ifdef CONFIG_X86_64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif Ouch...
Any good suggestion will be appreciated.
Quoted 6 lines
arch/x86/crypto/crc32c-intel.c line 31
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c +4
-0+static u32 crc32c_intel_le_hw_byte (u32 crc,unsigned char const *data,size_t length) +{ +while (length--) { + __asm__ __volatile__(Are all the underscores neccessary?
What's big impact if keep those underscores? Just keep ANSI C style.
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c line 172
Quoted 2 lines
arch/x86/crypto/crc32c-intel.c +1
-0+return -1 ;That's supposed to be errno, right?
Are you suggest "ENODEV"? It's a feature from the device but the device is exact here. And for the crc32c algorithm, there would be possible that several algorithms registered themselves in crypto and user will don't care which implementation will server him even the hardware accelerated implementation don't exist in this processor.
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Herbert Xu
maintainerQuoted 5 lines · Austin Zhang
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c line 172
Quoted 2 lines
arch/x86/crypto/crc32c-intel.c +1
-0+return -1 ;That's supposed to be errno, right?
Are you suggest "ENODEV"? It's a feature from the device but the device is exact here.
Yes I think this should be ENODEV to be consistent with the existing drivers such as padlock-aes.c.
I'll make that change in cryptodev.
Thanks,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 5 lines · Herbert Xu
Quoted 4 lines · Austin Zhang
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c line 172
Quoted 2 lines
arch/x86/crypto/crc32c-intel.c +1
-0+return -1 ;That's supposed to be errno, right?
Quoted 8 lines
Quoted 1 line
Are you suggest "ENODEV"? It's a feature from the device but the device is exact here.
Yes I think this should be ENODEV to be consistent with the existing drivers such as
padlock-aes.c.I'll make that change in cryptodev.
Thanks,
Thank you, Pavel and Herbert.
Herbert Xu
maintainerQuoted 11 lines · Herbert Xu
Quoted 5 lines · Austin Zhang
Quoted 4 lines
arch/x86/crypto/crc32c-intel.c line 172
Quoted 2 lines
arch/x86/crypto/crc32c-intel.c +1
-0+return -1 ;That's supposed to be errno, right?
Are you suggest "ENODEV"? It's a feature from the device but the device is exact here.
Yes I think this should be ENODEV to be consistent with the existing drivers such as
padlock-aes.c.I'll make that change in cryptodev.
In fact I'm going to kill that printk altogether since the fact that this feature doesn't exist isn't an error.
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 7 lines · Austin Zhang
Paval,
Thanks for your comments.
Quoted 1 line · Pavel Machek
Copyright / GPL?
Yes, as : ???+MODULE_LICENSE("GPL");
Well, it should normally go to comment at the beggining of file.
Pavel
Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Herbert Xu
maintainerQuoted 10 lines · Pavel Machek
Quoted 7 lines · Austin Zhang
Paval,
Thanks for your comments.
Quoted 1 line · Pavel Machek
Copyright / GPL?
Yes, as : ???+MODULE_LICENSE("GPL");
Well, it should normally go to comment at the beggining of file.
I've made the following change.
Thanks,
Signature · 5 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
--arch/x86/crypto/crc32c-intel.c +9 -0
@@ -5,6 +5,15 @@* http://www.intel.com/products/processor/manuals/ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual * Volume 2A: Instruction Set Reference, A-M +* +* Copyright (c) 2008 Austin Zhang <[email protected]> +* Copyright (c) 2008 Kent Liu <[email protected]> +* +* This program is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License as published by the Free +* Software Foundation; either version 2 of the License, or (at your option) +* any later version. +* */ #include <linux/init.h> #include <linux/module.h>
Quoted 4 lines · Austin Zhang
Revised by comments:
Add 'static' for limitation namespace;
Resend for fixing lines-folded by adjusting evolution config; (The patch was created against 2.6.27-rc1)
Applied to cryptodev-2.6. Thanks a lot Austin!
Signature · 8 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.htmlQuoted 7 lines · Austin Zhang
arch/x86/crypto/crc32c-intel.c +7
-0+#ifdef CONFIG_X86_64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif
[...]
arch/x86/crypto/crc32c-intel.c line 54
Quoted 13 lines
arch/x86/crypto/crc32c-intel.c +13
-0+static u32 __pure crc32c_intel_le_hw (u32 crc,unsigned char const *p,size_t len) +{ +unsigned int iquotient = len /SCALE_F ; +unsigned int iremainder = len %SCALE_F ; +unsigned long *ptmp = (unsigned long *)p; + +while (iquotient--) { + __asm__ __volatile__( +".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" + :"=S" (crc) + :"0" (crc),"c" (*ptmp) + ); + ptmp++;
I think you want to use
#define SCALE_F sizeof(unsigned long)
Since the loop iteration count etc depends on
ptmp++
which depends on the type being unsigned long.
Herbert Xu
maintainerQuoted 4 lines · Ulrich Drepper
I think you want to use
#define
SCALE_Fsizeof(unsigned long)
Yeah in general that's what we should do. However, this driver is specific to Intel x86 CPUs.
However if someone were to post a patch to do this I would happily apply it.
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 8 lines · Herbert Xu
Quoted 4 lines · Ulrich Drepper
I think you want to use
#define
SCALE_Fsizeof(unsigned long)Yeah in general that's what we should do. However, this driver is specific to Intel x86 CPUs.
I thought we support intel x86 cpus in both 32 and 64bit modes...?
Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Herbert Xu
maintainerQuoted 2 lines · Pavel Machek
I thought we support intel x86 cpus in both 32 and 64bit modes...?
Yes we do, but the original patch had ugly ifdefs that did the same thing.
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt