Skip to contentSkip to conversation

Using Intel CRC32 instruction to implement hardware accelerated CRC32c algorithm.]

PATCH In review austin zhang 2 participants 2 messages Jul 7, 2008

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Using Intel CRC32 instruction to implement hardware accelerated CRC32c algorithm.]

From NHM processor onward, Intel processors can support hardware accelerated CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum.
Byte code instructions are used for compiler compatibility. No MMX / XMM registers is involved in the implementation.

After basic testing with iSCSI and confirmed that the iSCSI head digest routines can be speeded up by 4x - 10x.

This patch is created against mainline 2.6.26-rc8

---
 include/asm-x86/cpufeature.h |    2 +
 include/asm-x86/crc32c-hw.h  |   63 +++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/processor.h  |    3 ++
 lib/libcrc32c.c              |   20 ++++++++++++-
 4 files changed, 86 insertions(+), 2 deletions(-)
include/asm-x86/cpufeature.h +2 -0
diff -Naurp linux-2.6-rc8/include/asm-x86/cpufeature.h linux-2.6-rc8-patch/include/asm-x86/cpufeature.h
@@ -90,6 +90,7 @@
 #define X86_FEATURE_CX16	(4*32+13) /* CMPXCHG16B */
 #define X86_FEATURE_XTPR	(4*32+14) /* Send Task Priority Messages */
 #define X86_FEATURE_DCA		(4*32+18) /* Direct Cache Access */
+#define X86_FEATURE_XMM4_2	(4*32+20) /* Streaming SIMD Extensions-4.2 */
 
 /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
 #define X86_FEATURE_XSTORE	(5*32+ 2) /* on-CPU RNG present (xstore insn) */
@@ -187,6 +188,7 @@ extern const char * const x86_power_flag
 #define cpu_has_gbpages		boot_cpu_has(X86_FEATURE_GBPAGES)
 #define cpu_has_arch_perfmon	boot_cpu_has(X86_FEATURE_ARCH_PERFMON)
 #define cpu_has_pat		boot_cpu_has(X86_FEATURE_PAT)
+#define cpu_has_xmm4_2		boot_cpu_has(X86_FEATURE_XMM4_2)
 
 #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
 # define cpu_has_invlpg		1
include/asm-x86/crc32c-hw.h +63 -0

63 lines not on this page · raw message

include/asm-x86/processor.h +3 -0
diff -Naurp linux-2.6-rc8/include/asm-x86/processor.h linux-2.6-rc8-patch/include/asm-x86/processor.h
@@ -777,6 +777,9 @@ extern char			ignore_fpu_irq;
 # define BASE_PREFETCH		"prefetcht0 (%1)"
 #endif
 
+/* Hardware provides CRC32C accelerated instruction */
+#define ARCH_HAS_CRC32C_HW	1
+
 /*
  * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
  *
lib/libcrc32c.c +18 -2
diff -Naurp linux-2.6-rc8/lib/libcrc32c.c linux-2.6-rc8-patch/lib/libcrc32c.c
@@ -34,6 +34,10 @@
 #include <linux/compiler.h>
 #include <linux/module.h>
 
+#ifdef ARCH_HAS_CRC32C_HW
+#include <asm/crc32c-hw.h>
+#endif
+
 MODULE_AUTHOR("Clay Haapala <[email protected]>");
 MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations");
 MODULE_LICENSE("GPL");
@@ -66,7 +70,7 @@ EXPORT_SYMBOL(crc32c_le);
  * of space and maintainability in keeping the two modules separate.
  */
 u32 __pure
-crc32c_le(u32 crc, unsigned char const *p, size_t len)
+crc32c_le_sw(u32 crc, unsigned char const *p, size_t len)
 {
 	int i;
 	while (len--) {
@@ -160,7 +164,7 @@ static const u32 crc32c_table[256] = {
  */
 
 u32 __pure
-crc32c_le(u32 crc, unsigned char const *data, size_t length)
+crc32c_le_sw(u32 crc, unsigned char const *data, size_t length)
 {
 	while (length--)
 		crc =
@@ -171,6 +175,18 @@ crc32c_le(u32 crc, unsigned char const *
 
 #endif	/* CRC_LE_BITS == 8 */
 
+u32 __pure
+crc32c_le(u32 crc, unsigned char const *p, size_t len)
+{
+#ifdef ARCH_HAS_CRC32C_HW
+	if (cpu_has_xmm4_2)
+		return crc32c_le_hw(crc, p, len);
+#endif
+
+	return crc32c_le_sw(crc, p, len);
+}
+
+
 EXPORT_SYMBOL(crc32c_be);
 
 #if CRC_BE_BITS == 1
maintainer
#1 lore raw
Quoted 7 lines · austin zhang

From NHM processor onward, Intel processors can support hardware accelerated CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum.
Byte code instructions are used for compiler compatibility. No MMX / XMM registers is involved in the implementation.

Quoted 7 lines

After basic testing with iSCSI and confirmed that the iSCSI head digest routines can be speeded up by 4x - 10x.

This patch is created against mainline 2.6.26-rc8

Thanks Austin! I love the idea :)

However, the way it's done in lib is a bit iffy. For a start testing an x86-specific variable at run-time is not a goer. In any case, supporting hardware implementations transparently is what the crypto API is meant to do so why don't we just use that.

I've done a grep on the users of crc32c. I think what we should do is:

1) Utilise the brand new crypto ahash interface (note that it's designed to support sync just as well as async despite the name) to rewrite the crypto/crc32c implementation such that one tfm can be used by multiple users. All that has to be done is to move the state from the tfm into the request object.

2) Convert all crc32c users to use the crypto interface and phase out lib/crc32c completely.

3) Add the Intel-specific crc32c implementation through the crypto API.

That way none of this iffy testing will be necessary. Even better, most users can share one common tfm and therefore there will only be one test for the CPU flag at boot time rather than every time it's used.

In fact, we could even skip 2) and reimplement lib/crc32c as a wrapper on the crypto crc32c interface with a shared tfm so you don't need to modify its existing users.

Cheers,

Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt