btrfs-progs: crypto: cleanup and prepare for new architectures

index 32914d360e63..a99efb932ff2 100644
@@ -15,37 +15,14 @@
  */
 
 #include <stdio.h>
+#ifdef __x86_64__
+#include <cpuid.h>
+#endif
 #include "common/cpu-utils.h"
 
 unsigned long __cpu_flags = CPU_FLAG_NONE;
 unsigned long __cpu_flags_orig = CPU_FLAG_NONE;
 
-#ifdef __x86_64__
-/*
- * Do manual cpuid as cpuid.h is not available for all versions.
- *
- * Plain cpuid(level) is __cpuidex(level, 0), otherwise look for the leaf/subleaf
- * modes at https://en.wikipedia.org/wiki/CPUID or compilers' cpuid.h.
- */
-
-#ifndef __cpuidex
-#define __cpuidex(leaf, subleaf, a, b, c, d)				\
-	__asm__ __volatile__ ("cpuid\n\t"				\
-			: "=a" (a), "=b" (b), "=c" (c), "=d" (d)	\
-			: "0" (leaf), "2" (subleaf))
-#endif
-
-#else
-
-#define __cpuidex(leaf, subleaf, a, b, c, d)				\
-	do {								\
-		a = 0;							\
-		b = 0;							\
-		c = 0;							\
-		d = 0;							\
-	} while(0)
-#endif
-
 #define FLAG(name)	if (__cpu_flags & CPU_FLAG_ ## name) printf(" " #name)
 void cpu_print_flags(void) {
 	printf("CPU flags: 0x%lx\n", __cpu_flags);

@@ -88,7 +65,7 @@ void cpu_detect_flags(void)
 		__cpu_flags |= CPU_FLAG_AVX2;
 
 	/* Flags unsupported by builtins */
-	__cpuidex(7, 0, a, b, c, d);
+	__cpuid_count(7, 0, a, b, c, d);
 	if (b & (1UL << 29))
 		__cpu_flags |= CPU_FLAG_SHA;
 

index f0a23f836741..9a3c9fc14eab 100644
@@ -18,6 +18,8 @@ static uint32_t (*crc32c_impl)(uint32_t crc, unsigned char const *data, uint32_t
 
 #ifdef __x86_64__
 
+#include <nmmintrin.h>
+
 /* asmlinkage */ unsigned int crc_pcl(const unsigned char *buffer, unsigned int len, unsigned int crc_init);
 static unsigned int crc32c_pcl(uint32_t crc, unsigned char const *data, uint32_t len) {
 	return crc_pcl(data, len, crc);

@@ -34,22 +36,19 @@ static unsigned int crc32c_pcl(uint32_t crc, unsigned char const *data, uint32_t
  * Volume 2A: Instruction Set Reference, A-M
  */
 #if  __SIZEOF_LONG__ == 8
-#define REX_PRE "0x48, "
 #define SCALE_F 8
+#define crc32c_hw_long(crc, v)	_mm_crc32_u64((crc), (v))
 #else
-#define REX_PRE
 #define SCALE_F 4
+#define crc32c_hw_long(crc, v)	_mm_crc32_u32((crc), (v))
 #endif
 
+__attribute__((target("sse4.2")))
 static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
 					uint32_t length)
 {
 	while (length--) {
-		__asm__ __volatile__(
-			".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
-			:"=S"(crc)
-			:"0"(crc), "c"(*data)
-		);
+		crc = _mm_crc32_u8(crc, *data);
 		data++;
 	}
 

@@ -61,6 +60,7 @@ static uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned char const *data,
  * Steps through buffer one byte at at time, calculates reflected crc using
  * table.
  */
+__attribute__((target("sse4.2")))
 static uint32_t crc32c_sse42(uint32_t crc, unsigned char const *data, uint32_t length)
 {
 	unsigned int iquotient = length / SCALE_F;

@@ -68,11 +68,7 @@ static uint32_t crc32c_sse42(uint32_t crc, unsigned char const *data, uint32_t l
 	unsigned long *ptmp = (unsigned long *)data;
 
 	while (iquotient--) {
-		__asm__ __volatile__(
-			".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
-			:"=S"(crc)
-			:"0"(crc), "c"(*ptmp)
-		);
+		crc = crc32c_hw_long(crc, *ptmp);
 		ptmp++;
 	}
 

index 6caa89f7d6f8..95ba63ee6025 100644
@@ -30,6 +30,7 @@
 #include "common/cpu-utils.h"
 
 #ifdef __x86_64__
+#include <x86intrin.h>
 static const int cycles_supported = 1;
 #else
 static const int cycles_supported = 0;

@@ -45,19 +46,10 @@ const int blocksize = 4096;
 int iterations = 100000;
 
 #ifdef __x86_64__
-static __always_inline unsigned long long rdtsc(void)
-{
-	unsigned low, high;
-
-        asm volatile("rdtsc" : "=a" (low), "=d" (high));
-
-        return (low | ((u64)(high) << 32));
-}
-
 static inline u64 read_tsc(void)
 {
-       asm volatile("mfence");
-       return rdtsc();
+	_mm_mfence();
+	return __rdtsc();
 }
 
 #define cpu_cycles()		read_tsc()

90 lines not on this page · raw message

281 lines not on this page · raw message

index a99efb932ff2..d8fbf664b998 100644
@@ -23,21 +23,21 @@
 unsigned long __cpu_flags = CPU_FLAG_NONE;
 unsigned long __cpu_flags_orig = CPU_FLAG_NONE;
 
-#define FLAG(name)	if (__cpu_flags & CPU_FLAG_ ## name) printf(" " #name)
+#define FLAG_X64(name)	if (__cpu_flags & CPU_FLAG_X64_ ## name) printf(" " #name)
 void cpu_print_flags(void) {
 	printf("CPU flags: 0x%lx\n", __cpu_flags);
 	printf("CPU features:");
-	FLAG(SSE2);
-	FLAG(SSSE3);
-	FLAG(SSE41);
-	FLAG(SSE42);
-	FLAG(PCLMUL);
-	FLAG(SHA);
-	FLAG(AVX);
-	FLAG(AVX2);
+	FLAG_X64(SSE2);
+	FLAG_X64(SSSE3);
+	FLAG_X64(SSE41);
+	FLAG_X64(SSE42);
+	FLAG_X64(PCLMUL);
+	FLAG_X64(SHA);
+	FLAG_X64(AVX);
+	FLAG_X64(AVX2);
 	putchar(10);
 }
-#undef FLAG
+#undef FLAG_X64
 
 #ifdef __x86_64__
 

@@ -48,26 +48,26 @@ void cpu_detect_flags(void)
 	__builtin_cpu_init();
 	__cpu_flags = CPU_FLAG_NONE;
 	if (__builtin_cpu_supports("sse2"))
-		__cpu_flags |= CPU_FLAG_SSE2;
+		__cpu_flags |= CPU_FLAG_X64_SSE2;
 	if (__builtin_cpu_supports("ssse3"))
-		__cpu_flags |= CPU_FLAG_SSSE3;
+		__cpu_flags |= CPU_FLAG_X64_SSSE3;
 	if (__builtin_cpu_supports("sse4.1"))
-		__cpu_flags |= CPU_FLAG_SSE41;
+		__cpu_flags |= CPU_FLAG_X64_SSE41;
 	if (__builtin_cpu_supports("sse4.2"))
-		__cpu_flags |= CPU_FLAG_SSE42;
+		__cpu_flags |= CPU_FLAG_X64_SSE42;
 #if HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL
 	if (__builtin_cpu_supports("pclmul"))
-		__cpu_flags |= CPU_FLAG_PCLMUL;
+		__cpu_flags |= CPU_FLAG_X64_PCLMUL;
 #endif
 	if (__builtin_cpu_supports("avx"))
-		__cpu_flags |= CPU_FLAG_AVX;
+		__cpu_flags |= CPU_FLAG_X64_AVX;
 	if (__builtin_cpu_supports("avx2"))
-		__cpu_flags |= CPU_FLAG_AVX2;
+		__cpu_flags |= CPU_FLAG_X64_AVX2;
 
 	/* Flags unsupported by builtins */
 	__cpuid_count(7, 0, a, b, c, d);
 	if (b & (1UL << 29))
-		__cpu_flags |= CPU_FLAG_SHA;
+		__cpu_flags |= CPU_FLAG_X64_SHA;
 
 	__cpu_flags_orig = __cpu_flags;
 }

index 21678dc6e8e7..97107f3f9086 100644
@@ -30,14 +30,14 @@
 
 enum cpu_feature {
 	ENUM_CPU_BIT(CPU_FLAG_NONE),
-	ENUM_CPU_BIT(CPU_FLAG_SSE2),
-	ENUM_CPU_BIT(CPU_FLAG_SSSE3),
-	ENUM_CPU_BIT(CPU_FLAG_SSE41),
-	ENUM_CPU_BIT(CPU_FLAG_SSE42),
-	ENUM_CPU_BIT(CPU_FLAG_PCLMUL),
-	ENUM_CPU_BIT(CPU_FLAG_SHA),
-	ENUM_CPU_BIT(CPU_FLAG_AVX),
-	ENUM_CPU_BIT(CPU_FLAG_AVX2),
+	ENUM_CPU_BIT(CPU_FLAG_X64_SSE2),
+	ENUM_CPU_BIT(CPU_FLAG_X64_SSSE3),
+	ENUM_CPU_BIT(CPU_FLAG_X64_SSE41),
+	ENUM_CPU_BIT(CPU_FLAG_X64_SSE42),
+	ENUM_CPU_BIT(CPU_FLAG_X64_PCLMUL),
+	ENUM_CPU_BIT(CPU_FLAG_X64_SHA),
+	ENUM_CPU_BIT(CPU_FLAG_X64_AVX),
+	ENUM_CPU_BIT(CPU_FLAG_X64_AVX2),
 };
 
 #undef ENUM_CPU_BIT

index 2c7e72ee2026..56774ccb04f9 100644
@@ -224,15 +224,15 @@ void blake2_init_accel(void)
 {
 	if (0);
 #if HAVE_CFLAG_mavx2 == 1
-	else if (cpu_has_feature(CPU_FLAG_AVX2))
+	else if (cpu_has_feature(CPU_FLAG_X64_AVX2))
 		blake2b_compress = blake2b_compress_avx2;
 #endif
 #if HAVE_CFLAG_msse41 == 1
-	else if (cpu_has_feature(CPU_FLAG_SSE41))
+	else if (cpu_has_feature(CPU_FLAG_X64_SSE41))
 		blake2b_compress = blake2b_compress_sse41;
 #endif
 #if HAVE_CFLAG_msse2 == 1
-	else if (cpu_has_feature(CPU_FLAG_SSE2))
+	else if (cpu_has_feature(CPU_FLAG_X64_SSE2))
 		blake2b_compress = blake2b_compress_sse2;
 #endif
 	else

index daf232c44e4d..0da4a1573c30 100644
@@ -335,11 +335,11 @@ static uint32_t crc32c_sse42(uint32_t crc, unsigned char const *data, uint32_t l
 void crc32c_init_accel(void)
 {
 	if (0) {
-	} else if (cpu_has_feature(CPU_FLAG_PCLMUL) &&
-		   cpu_has_feature(CPU_FLAG_SSE42)) {
+	} else if (cpu_has_feature(CPU_FLAG_X64_PCLMUL) &&
+		   cpu_has_feature(CPU_FLAG_X64_SSE42)) {
 		/* printf("CRC32C: pcl\n"); */
 		crc32c_impl = crc32c_pcl;
-	} else if (cpu_has_feature(CPU_FLAG_SSE42)) {
+	} else if (cpu_has_feature(CPU_FLAG_X64_SSE42)) {
 		/* printf("CRC32c: intel\n"); */
 		crc32c_impl = crc32c_sse42;
 	} else {

index 95ba63ee6025..5446161af997 100644
@@ -168,7 +168,7 @@ int main(int argc, char **argv) {
 	int iter;
 	char *filter = NULL;
 	struct contestant {
-		char name[16];
+		char name[20];
 		int (*digest)(const u8 *buf, size_t length, u8 *out);
 		int digest_size;
 		u64 cycles;

@@ -180,10 +180,10 @@ int main(int argc, char **argv) {
 		{ .name = "NULL-MEMCPY", .digest = hash_null_memcpy, .digest_size = 32 },
 		{ .name = "CRC32C-ref", .digest = hash_crc32c, .digest_size = 4,
 		  .cpu_flag = CPU_FLAG_NONE },
-		{ .name = "CRC32C-SSE42", .digest = hash_crc32c, .digest_size = 4,
-		  .cpu_flag = CPU_FLAG_SSE42 },
-		{ .name = "CRC32C-NI", .digest = hash_crc32c, .digest_size = 4,
-		  .cpu_flag = CPU_FLAG_PCLMUL },
+		{ .name = "CRC32C-X64-SSE42", .digest = hash_crc32c, .digest_size = 4,
+		  .cpu_flag = CPU_FLAG_X64_SSE42 },
+		{ .name = "CRC32C-X64-NI", .digest = hash_crc32c, .digest_size = 4,
+		  .cpu_flag = CPU_FLAG_X64_PCLMUL },
 		{ .name = "XXHASH", .digest = hash_xxhash, .digest_size = 8 },
 		{ .name = "SHA256-ref", .digest = hash_sha256, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BUILTIN + 1 },

@@ -197,8 +197,8 @@ int main(int argc, char **argv) {
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BOTAN + 1 },
 		{ .name = "SHA256-openssl", .digest = hash_sha256, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_OPENSSL + 1 },
-		{ .name = "SHA256-NI", .digest = hash_sha256, .digest_size = 32,
-		  .cpu_flag = CPU_FLAG_SHA, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
+		{ .name = "SHA256-X64-NI", .digest = hash_sha256, .digest_size = 32,
+		  .cpu_flag = CPU_FLAG_X64_SHA, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
 		{ .name = "BLAKE2-ref", .digest = hash_blake2b, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
 		{ .name = "BLAKE2-gcrypt", .digest = hash_blake2b, .digest_size = 32,

@@ -211,12 +211,12 @@ int main(int argc, char **argv) {
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BOTAN + 1 },
 		{ .name = "BLAKE2-openssl", .digest = hash_blake2b, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_OPENSSL + 1 },
-		{ .name = "BLAKE2-SSE2", .digest = hash_blake2b, .digest_size = 32,
-		  .cpu_flag = CPU_FLAG_SSE2, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
-		{ .name = "BLAKE2-SSE41", .digest = hash_blake2b, .digest_size = 32,
-		  .cpu_flag = CPU_FLAG_SSE41, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
-		{ .name = "BLAKE2-AVX2", .digest = hash_blake2b, .digest_size = 32,
-		  .cpu_flag = CPU_FLAG_AVX2, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
+		{ .name = "BLAKE2-X64-SSE2", .digest = hash_blake2b, .digest_size = 32,
+		  .cpu_flag = CPU_FLAG_X64_SSE2, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
+		{ .name = "BLAKE2-X64-SSE41", .digest = hash_blake2b, .digest_size = 32,
+		  .cpu_flag = CPU_FLAG_X64_SSE41, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
+		{ .name = "BLAKE2-X64-AVX2", .digest = hash_blake2b, .digest_size = 32,
+		  .cpu_flag = CPU_FLAG_X64_AVX2, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
 	};
 	int units = UNITS_CYCLES;
 

@@ -293,7 +293,7 @@ int main(int argc, char **argv) {
 		u64 total = 0;
 
 		if (c->cpu_flag != 0 && !cpu_has_feature(c->cpu_flag)) {
-			printf("%14s: no CPU support\n", c->name);
+			printf("%16s: no CPU support\n", c->name);
 			continue;
 		}
 		/* Backend not compiled in */

@@ -305,7 +305,7 @@ int main(int argc, char **argv) {
 		else if (filter && fnmatch(filter, c->name, FNM_CASEFOLD) != 0)
 			continue;
 
-		printf("%14s: ", c->name);
+		printf("%16s: ", c->name);
 		fflush(stdout);
 
 		if (c->cpu_flag) {

index e9c6eda976a4..2d9cc5da8df6 100644
@@ -435,18 +435,18 @@ static const struct hash_testspec test_spec[] = {
 		.cpu_flag = CPU_FLAG_NONE,
 		.hash = hash_crc32c,
 	}, {
-		.name = "CRC32C-SSE42",
+		.name = "CRC32C-X64-SSE42",
 		.digest_size = 4,
 		.testvec = crc32c_tv,
 		.count = ARRAY_SIZE(crc32c_tv),
-		.cpu_flag = CPU_FLAG_SSE42,
+		.cpu_flag = CPU_FLAG_X64_SSE42,
 		.hash = hash_crc32c,
 	}, {
-		.name = "CRC32C-NI",
+		.name = "CRC32C-X64-NI",
 		.digest_size = 4,
 		.testvec = crc32c_tv,
 		.count = ARRAY_SIZE(crc32c_tv),
-		.cpu_flag = CPU_FLAG_PCLMUL,
+		.cpu_flag = CPU_FLAG_X64_PCLMUL,
 		.hash = hash_crc32c
 	}, {
 		.name = "XXHASH",

@@ -503,11 +503,11 @@ static const struct hash_testspec test_spec[] = {
 		.hash = hash_sha256,
 		.backend = CRYPTOPROVIDER_OPENSSL + 1
 	}, {
-		.name = "SHA256-NI",
+		.name = "SHA256-X64-NI",
 		.digest_size = 32,
 		.testvec = sha256_tv,
 		.count = ARRAY_SIZE(sha256_tv),
-		.cpu_flag = CPU_FLAG_SHA,
+		.cpu_flag = CPU_FLAG_X64_SHA,
 		.hash = hash_sha256,
 		.backend = CRYPTOPROVIDER_BUILTIN + 1
 	}, {

@@ -559,27 +559,27 @@ static const struct hash_testspec test_spec[] = {
 		.hash = hash_blake2b,
 		.backend = CRYPTOPROVIDER_OPENSSL + 1
 	}, {
-		.name = "BLAKE2-SSE2",
+		.name = "BLAKE2-X64-SSE2",
 		.digest_size = 32,
 		.testvec = blake2b_256_tv,
 		.count = ARRAY_SIZE(blake2b_256_tv),
-		.cpu_flag = CPU_FLAG_SSE2,
+		.cpu_flag = CPU_FLAG_X64_SSE2,
 		.hash = hash_blake2b,
 		.backend = CRYPTOPROVIDER_BUILTIN + 1
 	}, {
-		.name = "BLAKE2-SSE41",
+		.name = "BLAKE2-X64-SSE41",
 		.digest_size = 32,
 		.testvec = blake2b_256_tv,
 		.count = ARRAY_SIZE(blake2b_256_tv),
-		.cpu_flag = CPU_FLAG_SSE41,
+		.cpu_flag = CPU_FLAG_X64_SSE41,
 		.hash = hash_blake2b,
 		.backend = CRYPTOPROVIDER_BUILTIN + 1
 	}, {
-		.name = "BLAKE2-AVX2",
+		.name = "BLAKE2-X64-AVX2",
 		.digest_size = 32,
 		.testvec = blake2b_256_tv,
 		.count = ARRAY_SIZE(blake2b_256_tv),
-		.cpu_flag = CPU_FLAG_AVX2,
+		.cpu_flag = CPU_FLAG_X64_AVX2,
 		.hash = hash_blake2b,
 		.backend = CRYPTOPROVIDER_BUILTIN + 1
 	}

index e2075de5abfb..69e7ce5b9bd2 100644
@@ -110,7 +110,7 @@ static void sha256_process_x86_dispatch(SHA256Context *context)
 void sha256_init_accel(void)
 {
 #if HAVE_CFLAG_msha == 1
-	if (cpu_has_feature(CPU_FLAG_SHA))
+	if (cpu_has_feature(CPU_FLAG_X64_SHA))
 		sha256_process_message_block = sha256_process_x86_dispatch;
 	else
 #endif

index d8fbf664b998..e933b8322dc5 100644
@@ -15,10 +15,8 @@
  */
 
 #include <stdio.h>
-#ifdef __x86_64__
-#include <cpuid.h>
-#endif
 #include "common/cpu-utils.h"
+#include "common/messages.h"
 
 unsigned long __cpu_flags = CPU_FLAG_NONE;
 unsigned long __cpu_flags_orig = CPU_FLAG_NONE;

@@ -39,14 +37,12 @@ void cpu_print_flags(void) {
 }
 #undef FLAG_X64
 
-#ifdef __x86_64__
 
 void cpu_detect_flags(void)
 {
-	unsigned int a, b, c, d;
-
-	__builtin_cpu_init();
 	__cpu_flags = CPU_FLAG_NONE;
+#ifdef __x86_64__
+	__builtin_cpu_init();
 	if (__builtin_cpu_supports("sse2"))
 		__cpu_flags |= CPU_FLAG_X64_SSE2;
 	if (__builtin_cpu_supports("ssse3"))

@@ -58,39 +54,27 @@ void cpu_detect_flags(void)
 #if HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL
 	if (__builtin_cpu_supports("pclmul"))
 		__cpu_flags |= CPU_FLAG_X64_PCLMUL;
-#endif
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL */
 	if (__builtin_cpu_supports("avx"))
 		__cpu_flags |= CPU_FLAG_X64_AVX;
 	if (__builtin_cpu_supports("avx2"))
 		__cpu_flags |= CPU_FLAG_X64_AVX2;
-
-	/* Flags unsupported by builtins */
-	__cpuid_count(7, 0, a, b, c, d);
-	if (b & (1UL << 29))
+#if HAVE___BUILTIN_CPU_SUPPORTS__SHA
+	if (__builtin_cpu_supports("sha"))
 		__cpu_flags |= CPU_FLAG_X64_SHA;
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__SHA */
+#endif /* __x86_64__ */
 
 	__cpu_flags_orig = __cpu_flags;
 }
 
 void cpu_set_level(unsigned long topbit)
 {
-	if (topbit)
-		__cpu_flags &= (topbit << 1) - 1;
-	else
-		__cpu_flags = CPU_FLAG_NONE;
+	UASSERT(topbit);
+	__cpu_flags &= (topbit << 1) - 1;
 }
 
 void cpu_reset_level(void)
 {
 	__cpu_flags = __cpu_flags_orig;
 }
-
-#else
-
-void cpu_detect_flags() { }
-
-void cpu_set_level(unsigned long topbit) { }
-
-void cpu_reset_level(void) { }
-
-#endif

index 5446161af997..b51ccbb19284 100644
@@ -182,8 +182,10 @@ int main(int argc, char **argv) {
 		  .cpu_flag = CPU_FLAG_NONE },
 		{ .name = "CRC32C-X64-SSE42", .digest = hash_crc32c, .digest_size = 4,
 		  .cpu_flag = CPU_FLAG_X64_SSE42 },
+#if HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL
 		{ .name = "CRC32C-X64-NI", .digest = hash_crc32c, .digest_size = 4,
 		  .cpu_flag = CPU_FLAG_X64_PCLMUL },
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL */
 		{ .name = "XXHASH", .digest = hash_xxhash, .digest_size = 8 },
 		{ .name = "SHA256-ref", .digest = hash_sha256, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BUILTIN + 1 },

@@ -197,8 +199,10 @@ int main(int argc, char **argv) {
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BOTAN + 1 },
 		{ .name = "SHA256-openssl", .digest = hash_sha256, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_OPENSSL + 1 },
+#if HAVE___BUILTIN_CPU_SUPPORTS__SHA
 		{ .name = "SHA256-X64-NI", .digest = hash_sha256, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_X64_SHA, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__SHA */
 		{ .name = "BLAKE2-ref", .digest = hash_blake2b, .digest_size = 32,
 		  .cpu_flag = CPU_FLAG_NONE, .backend = CRYPTOPROVIDER_BUILTIN + 1 },
 		{ .name = "BLAKE2-gcrypt", .digest = hash_blake2b, .digest_size = 32,

index 2d9cc5da8df6..3b1b79eae138 100644
@@ -441,14 +441,19 @@ static const struct hash_testspec test_spec[] = {
 		.count = ARRAY_SIZE(crc32c_tv),
 		.cpu_flag = CPU_FLAG_X64_SSE42,
 		.hash = hash_crc32c,
-	}, {
+	},
+
+#if HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL
+	{
 		.name = "CRC32C-X64-NI",
 		.digest_size = 4,
 		.testvec = crc32c_tv,
 		.count = ARRAY_SIZE(crc32c_tv),
 		.cpu_flag = CPU_FLAG_X64_PCLMUL,
 		.hash = hash_crc32c
-	}, {
+	},
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__PCLMUL */
+	{
 		.name = "XXHASH",
 		.digest_size = 8,
 		.testvec = xxhash64_tv,

@@ -502,7 +507,9 @@ static const struct hash_testspec test_spec[] = {
 		.cpu_flag = CPU_FLAG_NONE,
 		.hash = hash_sha256,
 		.backend = CRYPTOPROVIDER_OPENSSL + 1
-	}, {
+	},
+#ifdef HAVE___BUILTIN_CPU_SUPPORTS__SHA
+	{
 		.name = "SHA256-X64-NI",
 		.digest_size = 32,
 		.testvec = sha256_tv,

@@ -510,7 +517,9 @@ static const struct hash_testspec test_spec[] = {
 		.cpu_flag = CPU_FLAG_X64_SHA,
 		.hash = hash_sha256,
 		.backend = CRYPTOPROVIDER_BUILTIN + 1
-	}, {
+	},
+#endif /* HAVE___BUILTIN_CPU_SUPPORTS__SHA */
+	{
 		.name = "BLAKE2-ref",
 		.digest_size = 32,
 		.testvec = blake2b_256_tv,

index e933b8322dc5..76959c0b729f 100644
@@ -16,10 +16,8 @@
 
 #include <stdio.h>
 #include "common/cpu-utils.h"
-#include "common/messages.h"
 
 unsigned long __cpu_flags = CPU_FLAG_NONE;
-unsigned long __cpu_flags_orig = CPU_FLAG_NONE;
 
 #define FLAG_X64(name)	if (__cpu_flags & CPU_FLAG_X64_ ## name) printf(" " #name)
 void cpu_print_flags(void) {

@@ -64,17 +62,4 @@ void cpu_detect_flags(void)
 		__cpu_flags |= CPU_FLAG_X64_SHA;
 #endif /* HAVE___BUILTIN_CPU_SUPPORTS__SHA */
 #endif /* __x86_64__ */
-
-	__cpu_flags_orig = __cpu_flags;
-}
-
-void cpu_set_level(unsigned long topbit)
-{
-	UASSERT(topbit);
-	__cpu_flags &= (topbit << 1) - 1;
-}
-
-void cpu_reset_level(void)
-{
-	__cpu_flags = __cpu_flags_orig;
 }

index 97107f3f9086..2ca195567b84 100644
@@ -44,12 +44,9 @@ enum cpu_feature {
 
 /* Private but in public header to allow inlining */
 extern unsigned long __cpu_flags;
-extern unsigned long __cpu_flags_orig;
 
 /* Public API */
 void cpu_detect_flags(void);
-void cpu_set_level(unsigned long topbit);
-void cpu_reset_level(void);
 void cpu_print_flags(void);
 
 static inline bool cpu_has_feature(enum cpu_feature f)

index 052afe34a651..c4cc079ba04e 100644
@@ -17,6 +17,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include "common/cpu-utils.h"
 
 #if defined(_MSC_VER)
 #define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop))

@@ -91,6 +92,7 @@ extern "C" {
   int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen );
 
   void blake2_init_accel(void);
+  void blake2_force_accel(enum cpu_feature f);
 
   /* Export optimized versions to silent -Wmissing-prototypes warnings. */
   void blake2b_compress_avx2( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] );

index 56774ccb04f9..e8d2adb0498e 100644
@@ -239,6 +239,30 @@ void blake2_init_accel(void)
 		blake2b_compress = blake2b_compress_ref;
 }
 
+void blake2_force_accel(enum cpu_feature f)
+{
+	switch (f) {
+#if HAVE_CFLAG_mavx2 == 1
+	case CPU_FLAG_X64_AVX2:
+		blake2b_compress = blake2b_compress_avx2;
+		break;
+#endif
+#if HAVE_CFLAG_msse41 == 1
+	case CPU_FLAG_X64_SSE41:
+		blake2b_compress = blake2b_compress_sse41;
+		break;
+#endif
+#if HAVE_CFLAG_msse2 == 1
+	case CPU_FLAG_X64_SSE2:
+		blake2b_compress = blake2b_compress_sse2;
+		break;
+#endif
+	default:
+		blake2b_compress = blake2b_compress_ref;
+		break;
+	}
+}
+
 int blake2b_update( blake2b_state *S, const void *pin, size_t inlen )
 {
   const unsigned char * in = (const unsigned char *)pin;

54 lines not on this page · raw message

index f990177147e5..dd427893cf99 100644
@@ -20,9 +20,11 @@
 #define __CRC32C__
 
 #include <inttypes.h>
+#include "common/cpu-utils.h"
 
 uint32_t crc32c_le(uint32_t seed, unsigned char const *data, uint32_t length);
 void crc32c_init_accel(void);
+void crc32c_force_accel(enum cpu_feature f);
 
 #define crc32c(seed, data, length) crc32c_le(seed, (unsigned char const *)data, length)
 

index b51ccbb19284..a26959ae66e8 100644
@@ -312,10 +312,8 @@ int main(int argc, char **argv) {
 		printf("%16s: ", c->name);
 		fflush(stdout);
 
-		if (c->cpu_flag) {
-			cpu_set_level(c->cpu_flag);
-			hash_init_accel();
-		}
+		if (c->cpu_flag)
+			hash_force_accel(c->cpu_flag);
 		tstart = get_time();
 		start = get_cycles(units);
 		for (iter = 0; iter < iterations; iter++) {

@@ -327,7 +325,6 @@ int main(int argc, char **argv) {
 		tend = get_time();
 		c->cycles = end - start;
 		c->time = tend - tstart;
-		cpu_reset_level();
 
 		if (units == UNITS_CYCLES || units == UNITS_PERF)
 			total = c->cycles;

index 3b1b79eae138..c190a7a01b95 100644
@@ -617,12 +617,9 @@ static int test_hash(const struct hash_testspec *spec)
 			header = true;
 		}
 
-		if (spec->cpu_flag) {
-			cpu_set_level(spec->cpu_flag);
-			hash_init_accel();
-		}
+		if (spec->cpu_flag)
+			hash_force_accel(spec->cpu_flag);
 		ret = spec->hash((const u8 *)vec->plaintext, vec->psize, csum);
-		cpu_reset_level();
 		if (ret < 0) {
 			error("hash %s = %d", spec->name, ret);
 			return 1;

index 4af7b04a2507..675051873ef9 100644
@@ -49,6 +49,15 @@ int hash_xxhash(const u8 *buf, size_t length, u8 *out)
 	return 0;
 }
 
+void hash_force_accel(enum cpu_feature f)
+{
+	crc32c_force_accel(f);
+#if CRYPTOPROVIDER_BUILTIN == 1
+	blake2_force_accel(f);
+	sha256_force_accel(f);
+#endif
+}
+
 /*
  * Implementations of cryptographic primitives
  */

index 4f8f668ad260..11d977922039 100644
@@ -18,6 +18,7 @@
 #define CRYPTO_HASH_H
 
 #include "kerncompat.h"
+#include "common/cpu-utils.h"
 #include <stddef.h>
 
 #define CRYPTO_HASH_SIZE_MAX	32

@@ -28,6 +29,7 @@ int hash_sha256(const u8 *buf, size_t length, u8 *out);
 int hash_blake2b(const u8 *buf, size_t length, u8 *out);
 
 void hash_init_accel(void);
+void hash_force_accel(enum cpu_feature f);
 void hash_init_crc32c(void);
 
 #endif

index ea387c212dc6..0292b292f357 100644
@@ -71,6 +71,7 @@
  */
 
 #include <stdint.h>
+#include "common/cpu-utils.h"
 /*
  * If you do not have the ISO standard stdint.h header file, then you
  * must typedef the following:

@@ -210,6 +211,7 @@ extern int hmacResult(HMACContext *context,
                       uint8_t digest[USHAMaxHashSize]);
 
 void sha256_init_accel(void);
+void sha256_force_accel(enum cpu_feature f);
 
 /* Export for optimized version to silent -Wmissing-prototypes. */
 void sha256_process_x86(uint32_t state[8], const uint8_t data[], uint32_t length);

index 69e7ce5b9bd2..1df03825fa8f 100644
@@ -110,11 +110,23 @@ static void sha256_process_x86_dispatch(SHA256Context *context)
 void sha256_init_accel(void)
 {
 #if HAVE_CFLAG_msha == 1
-	if (cpu_has_feature(CPU_FLAG_X64_SHA))
+	if (cpu_has_feature(CPU_FLAG_X64_SHA)) {
 		sha256_process_message_block = sha256_process_x86_dispatch;
-	else
+		return;
+	}
 #endif
-		sha256_process_message_block = SHA224_256ProcessMessageBlock;
+	sha256_process_message_block = SHA224_256ProcessMessageBlock;
+}
+
+void sha256_force_accel(enum cpu_feature f)
+{
+#if HAVE_CFLAG_msha == 1
+	if (f == CPU_FLAG_X64_SHA) {
+		sha256_process_message_block = sha256_process_x86_dispatch;
+		return;
+	}
+#endif
+	sha256_process_message_block = SHA224_256ProcessMessageBlock;
 }
 
 /*

index a26959ae66e8..9a7d81d582fb 100644
@@ -27,7 +27,9 @@
 #endif
 #include "crypto/hash.h"
 #include "common/messages.h"
+#include "kernel-lib/sizes.h"
 #include "common/cpu-utils.h"
+#include "kernel-shared/uapi/btrfs_tree.h"
 
 #ifdef __x86_64__
 #include <x86intrin.h>

@@ -42,7 +44,15 @@ enum {
 	UNITS_PERF,
 };
 
-const int blocksize = 4096;
+/*
+ * Default sizes:
+ * - 16, 255	btrfs_name_hash() on short and max name length
+ * - 4096	data checksum with 4K sectorsize
+ * - 16384	metadata checksum with 16K nodesize, the difference from the
+ *		real size (16K - 32, no csum header) is minimal
+ */
+static const size_t sizes[] = { SZ_16, BTRFS_NAME_LEN, SZ_4K, SZ_16K };
+
 int iterations = 100000;
 
 #ifdef __x86_64__

@@ -162,10 +172,8 @@ static const char *units_to_str(int units)
 }
 
 int main(int argc, char **argv) {
-	u8 buf[blocksize];
+	u8 *buf;
 	u8 hash[32];
-	int idx;
-	int iter;
 	char *filter = NULL;
 	struct contestant {
 		char name[20];

@@ -282,69 +290,77 @@ int main(int argc, char **argv) {
 			iterations = 1;
 	}
 
-	memset(buf, 0, 4096);
+	buf = malloc(SZ_16K);
+	if (!buf) {
+		error_mem("buffer");
+		return 1;
+	}
 
-	printf("Block size:     %d\n", blocksize);
 	printf("Iterations:     %d\n", iterations);
 	printf("Implementation: %s\n", CRYPTOPROVIDER);
 	printf("Units:          %s\n", units_to_desc(units));
-	printf("\n");
 
-	for (idx = 0; idx < ARRAY_SIZE(contestants); idx++) {
-		struct contestant *c = &contestants[idx];
-		u64 start, end;
-		u64 tstart, tend;
-		u64 total = 0;
+	for (int i = 0; i < ARRAY_SIZE(sizes); i++) {
+		size_t size = sizes[i];
 
-		if (c->cpu_flag != 0 && !cpu_has_feature(c->cpu_flag)) {
-			printf("%16s: no CPU support\n", c->name);
-			continue;
+		printf("\nBlock size:     %zu\n", size);
+		for (int idx = 0; idx < ARRAY_SIZE(contestants); idx++) {
+			struct contestant *c = &contestants[idx];
+			u64 start, end;
+			u64 tstart, tend;
+			u64 total = 0;
+
+			if (c->cpu_flag != 0 && !cpu_has_feature(c->cpu_flag)) {
+				printf("%16s: no CPU support\n", c->name);
+				continue;
+			}
+			/* Backend not compiled in */
+			if (c->backend == 1)
+				continue;
+
+			if (c->digest == hash_null_memcpy)
+				/* Always run NULL-MEMCPY to warm up memory. */;
+			else if (filter && fnmatch(filter, c->name, FNM_CASEFOLD) != 0)
+				continue;
+
+			printf("%16s: ", c->name);
+			fflush(stdout);
+
+			if (c->cpu_flag)
+				hash_force_accel(c->cpu_flag);
+			tstart = get_time();
+			start = get_cycles(units);
+			for (int iter = 0; iter < iterations; iter++) {
+				memset(buf, iter & 0xFF, size);
+				memset(hash, 0, 32);
+				c->digest(buf, size, hash);
+			}
+			end = get_cycles(units);
+			tend = get_time();
+			c->cycles = end - start;
+			c->time = tend - tstart;
+
+			if (units == UNITS_CYCLES || units == UNITS_PERF)
+				total = c->cycles;
+			if (units == UNITS_TIME)
+				total = c->time;
+
+			printf("%s: %12llu, %s/i %8llu",
+					units_to_str(units), total,
+					units_to_str(units), total / iterations);
+			if (idx > 0) {
+				float t;
+				float mb;
+
+				t = (float)c->time / 1000 / 1000 / 1000;
+				mb = (float)size * iterations / 1024 / 1024;
+				printf(", %12.3f MiB/s", mb / t);
+			}
+			putchar('\n');
 		}
-		/* Backend not compiled in */
-		if (c->backend == 1)
-			continue;
-
-		if (c->digest == hash_null_memcpy)
-			/* Always run NULL-MEMCPY to warm up memory. */;
-		else if (filter && fnmatch(filter, c->name, FNM_CASEFOLD) != 0)
-			continue;
-
-		printf("%16s: ", c->name);
-		fflush(stdout);
-
-		if (c->cpu_flag)
-			hash_force_accel(c->cpu_flag);
-		tstart = get_time();
-		start = get_cycles(units);
-		for (iter = 0; iter < iterations; iter++) {
-			memset(buf, iter & 0xFF, blocksize);
-			memset(hash, 0, 32);
-			c->digest(buf, blocksize, hash);
-		}
-		end = get_cycles(units);
-		tend = get_time();
-		c->cycles = end - start;
-		c->time = tend - tstart;
-
-		if (units == UNITS_CYCLES || units == UNITS_PERF)
-			total = c->cycles;
-		if (units == UNITS_TIME)
-			total = c->time;
-
-		printf("%s: %12llu, %s/i %8llu",
-				units_to_str(units), total,
-				units_to_str(units), total / iterations);
-		if (idx > 0) {
-			float t;
-			float mb;
-
-			t = (float)c->time / 1000 / 1000 / 1000;
-			mb = blocksize * iterations / 1024 / 1024;
-			printf(", %12.3f MiB/s", mb / t);
-		}
-		putchar('\n');
 	}
 	perf_finish();
+	free(buf);
 	free(filter);
 
 	return 0;

220 lines not on this page · raw message