crypto: Add PCLMULQDQ accelerated GHASH implementation
Subsystems (2)
PATCH 1/1 crypto: Add PCLMULQDQ accelerated GHASH implementation
PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:
Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.
v4:
- Fix some style issues.
v3:
- Revise GHASH implementation, performance increase about 2x.
--- arch/x86/crypto/Makefile | 3 arch/x86/crypto/ghash-clmulni-intel_asm.S | 157 +++++++++++++ arch/x86/crypto/ghash-clmulni-intel_glue.c | 333 +++++++++++++++++++++++++++++ arch/x86/include/asm/cpufeature.h | 1 crypto/Kconfig | 8 crypto/cryptd.c | 7 include/crypto/cryptd.h | 1 7 files changed, 510 insertions(+) create mode 100644 arch/x86/crypto/ghash-clmulni-intel_asm.S create mode 100644 arch/x86/crypto/ghash-clmulni-intel_glue.c
arch/x86/crypto/Makefile +3 -0
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o +obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
@@ -24,3 +25,5 @@ twofish-x86_64-y := twofish-x86_64-asm_6 salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o + +ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
arch/x86/crypto/ghash-clmulni-intel_asm.S new file +157 -0
157 lines not on this page · raw message
arch/x86/crypto/ghash-clmulni-intel_glue.c new file +333 -0
333 lines not on this page · raw message
arch/x86/include/asm/cpufeature.h +1 -0
@@ -247,6 +247,7 @@ extern const char * const x86_power_flag#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC)#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE)#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) +#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ)#if defined(CONFIG_X86_INVLPG ) || defined(CONFIG_X86_64 )# define cpu_has_invlpg 1
crypto/Kconfig +8 -0
@@ -440,6 +440,14 @@ config CRYPTO_WP512 See also: <http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html> +config CRYPTO_GHASH_CLMUL_NI_INTEL + tristate "GHASH digest algorithm (CLMUL-NI accelerated)" + select CRYPTO_SHASH + select CRYPTO_CRYPTD + help + GHASH is message digest algorithm for GCM (Galois/Counter Mode). + The implementation is accelerated by CLMUL-NI of Intel. + comment "Ciphers" config CRYPTO_AES
crypto/cryptd.c +7 -0
@@ -711,6 +711,13 @@ struct crypto_shash *cryptd_ahash_child( }EXPORT_SYMBOL_GPL (cryptd_ahash_child); +struct shash_desc *cryptd_shash_desc (struct ahash_request *req) +{ +struct cryptd_hash_request_ctx *rctx =ahash_request_ctx (req); +return &rctx->desc; +} +EXPORT_SYMBOL_GPL (cryptd_shash_desc); +void cryptd_free_ahash (struct cryptd_ahash *tfm) {crypto_free_ahash (&tfm->base);
include/crypto/cryptd.h +1 -0
@@ -39,6 +39,7 @@ static inline struct cryptd_ahash *__crystruct cryptd_ahash *cryptd_alloc_ahash (const char *alg_name,u32 type,u32 mask);struct crypto_shash *cryptd_ahash_child (struct cryptd_ahash *tfm); +struct shash_desc *cryptd_shash_desc (struct ahash_request *req);void cryptd_free_ahash (struct cryptd_ahash *tfm);#endif
Herbert Xu
maintainerQuoted 11 lines · Huang Ying
PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:
Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as
crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.
Quoted 7 lines
v4:
- Fix some style issues.v3:
- Revise GHASH implementation, performance increase about 2x.Signed-off-by Huang Ying <[email protected]>
Patch applied to cryptodev. Thanks!
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 15 lines · On Mon, 19 Oct 2009 11:53:33 +0900 Herbert Xu <[email protected]>…
Quoted 14 lines · Huang Ying
PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, carry-less multiplication. More information about PCLMULQDQ can be found at:
Because PCLMULQDQ changes XMM state, its usage must be enclosed with kernel_fpu_begin/end, which can be used only in process context, the acceleration is implemented as
crypto_ahash. That is, request in soft IRQ context will be defered to the cryptd kernel thread.v4:
- Fix some style issues.
Quoted 6 lines
Quoted 4 lines
v3:
- Revise GHASH implementation, performance increase about 2x.Signed-off-by Huang Ying <[email protected]>Patch applied to cryptodev. Thanks!
x86_64 allmodconfig, GNU assembler 2.16.1:
arch/x86/crypto/ghash-clmulni-intel_asm.S: Assembler messages:arch/x86/crypto/ghash-clmulni-intel_asm.S:103: Error: no such instruction: pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:105: Error: no such instruction: pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:119: Error: no such instruction: pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:123: Error: no such instruction: pshufb %xmm5,%xmm6' arch/x86/crypto/ghash-clmulni-intel_asm.S:130: Error: no such instruction: pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:143: Error: no such instruction: pshufb %xmm5,%xmm0'
Herbert Xu
maintainerQuoted 10 lines · Andrew Morton
x86_64allmodconfig, GNU assembler 2.16.1:
arch/x86/crypto/ghash-clmulni-intel_asm.S: Assembler messages:arch/x86/crypto/ghash-clmulni-intel_asm.S:103: Error: no such instruction:pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:105: Error: no such instruction:pshufb %xmm5,%xmm0'arch/x86/crypto/ghash-clmulni-intel_asm.S:119: Error: no such instruction:pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:123: Error: no such instruction:pshufb %xmm5,%xmm6'arch/x86/crypto/ghash-clmulni-intel_asm.S:130: Error: no such instruction:pshufb %xmm5,%xmm0' arch/x86/crypto/ghash-clmulni-intel_asm.S:143: Error: no such instruction:pshufb %xmm5,%xmm0'
This patch should fix it.
commit 2d06ef7f42ed8c9969c9aa84e95df5d5c6378327
Author: Herbert Xu <[email protected]> Date: Sun Nov 1 12:49:44 2009 -0500
crypto: ghash-intel - Hard-code pshufb
Old gases don't have a clue what pshufb stands for so we have to hard-code it for now.
arch/x86/crypto/ghash-clmulni-intel_asm.S +12 -6
@@ -100,9 +100,11 @@ ENTRY(clmul_ghash_mul) movups (%rdi), DATA movups (%rsi), SHASH movaps .Lbswap_mask, BSWAP - pshufb BSWAP, DATA + # pshufb BSWAP, DATA + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 call __clmul_gf128mul_ble - pshufb BSWAP, DATA + # pshufb BSWAP, DATA + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 movups DATA, (%rdi) ret
@@ -116,18 +118,21 @@ ENTRY(clmul_ghash_update) movaps .Lbswap_mask, BSWAP movups (%rdi), DATA movups (%rcx), SHASH - pshufb BSWAP, DATA + # pshufb BSWAP, DATA + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 .align 4 .Lupdate_loop: movups (%rsi), IN1 - pshufb BSWAP, IN1 + # pshufb BSWAP, IN1 + .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 pxor IN1, DATA call __clmul_gf128mul_ble sub $16, %rdx add $16, %rsi cmp $16, %rdx jge .Lupdate_loop - pshufb BSWAP, DATA + # pshufb BSWAP, DATA + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 movups DATA, (%rdi) .Lupdate_just_ret: ret
@@ -140,7 +145,8 @@ ENTRY(clmul_ghash_update) ENTRY(clmul_ghash_setkey) movaps .Lbswap_mask, BSWAP movups (%rsi), %xmm0 - pshufb BSWAP, %xmm0 + # pshufb BSWAP, %xmm0 + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 movaps %xmm0, %xmm1 psllq $1, %xmm0 psrlq $63, %xmm1
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtarch/x86/crypto/ghash-clmulni-intel_asm.S ENTRY(clmul_ghash_mul) · line 104
Quoted 3 lines · Herbert Xu
arch/x86/crypto/ghash-clmulni-intel_asm.S +2
-1- pshufb BSWAP, DATA + # pshufb BSWAP, DATA + .byte 0x66, 0x0f, 0x38, 0x00, 0xc5
A cleanup request: mind creating two macros for this PSHUFB MMX/SSE instruction in arch/x86/include/asm/i387.h, instead of open-coding the .byte sequences in ~6 places?
( After the .33 merge window we'll collect such instruction format
knowledge in arch/x86/include/asm/inst.h. That file is not upstream yet so i387.h will do for now for FPU/SSE instructions. )
Thanks,
Ingo
Herbert Xu
maintainerQuoted 4 lines · Ingo Molnar
A cleanup request: mind creating two macros for this PSHUFB MMX/SSE instruction in
arch/x86/include/asm/i387.h, instead of open-coding the .byte sequences in ~6 places?
I had a go at doing that, but it seems that i387.h isn't really meant to be included in an asm file at this point :)
Quoted 3 lines
( After the .33 merge window we'll collect such instruction format
knowledge inarch/x86/include/asm/inst.h. That file is not upstream yet soi387.hwill do for now for FPU/SSE instructions. )
I'm happy to revisit this once inst.h exists.
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 8 lines · Herbert Xu
Quoted 4 lines · Ingo Molnar
A cleanup request: mind creating two macros for this PSHUFB MMX/SSE instruction in
arch/x86/include/asm/i387.h, instead of open-coding the .byte sequences in ~6 places?I had a go at doing that, but it seems that
i387.hisn't really meant to be included in an asm file at this point :)
Please use the standard construct and put an #ifndef __ASSEMBLY__ around it.
Quoted 5 lines
Quoted 3 lines
( After the .33 merge window we'll collect such instruction format
knowledge inarch/x86/include/asm/inst.h. That file is not upstream yet soi387.hwill do for now for FPU/SSE instructions. )I'm happy to revisit this once
inst.hexists.
No reason to not do most of the change first though, the way i suggested it.
Ingo
Herbert Xu
maintainerQuoted 3 lines · Ingo Molnar
Please use the standard construct and put an #ifndef
__ASSEMBLY__around it.
You mean like this?
arch/x86/include/asm/i387.h +8 -0
@@ -10,6 +10,13 @@#ifndef _ASM_X86_I387_H#define _ASM_X86_I387_H +#ifdef __ASSEMBLY__ + +#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 +#define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 + +#else +#include <linux/sched.h> #include <linux/kernel_stat.h> #include <linux/regset.h>
@@ -411,4 +418,5 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } +#endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_I387_H */
Cheers,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 26 lines · Herbert Xu
Quoted 3 lines · Ingo Molnar
Please use the standard construct and put an #ifndef
__ASSEMBLY__around it.You mean like this?
arch/x86/include/asm/i387.h +7
-0@@ -10,6 +10,13 @@#ifndef _ASM_X86_I387_H#define _ASM_X86_I387_H +#ifdef __ASSEMBLY__ + +#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 +#define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 + +#else +#include <linux/sched.h> #include <linux/kernel_stat.h> #include <linux/regset.h>
arch/x86/include/asm/i387.h static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) · line 422
Quoted 5 lines
arch/x86/include/asm/i387.h +1
-0@@ -411,4 +418,5 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } +#endif /* __ASSEMBLY__ */ #endif /* _ASM_X86_I387_H */
Yeah. Or just a single block of:
#ifndef __ASSEMBLY__ ... #endif /* __ASSEMBLY__ */
around the C bits - anything outside that is good for assembly as well.
Ingo
Quoted 9 lines · Ingo Molnar
Yeah. Or just a single block of:
#ifndef __ASSEMBLY__ ... #endif /* __ASSEMBLY__ */around the C bits - anything outside that is good for assembly as well.
OK I'll throw this into cryptodev:
commit 3b0d65969b549b796abc6f0230f6142fed365d49
Author: Herbert Xu <[email protected]> Date: Tue Nov 3 09:11:15 2009 -0500
crypto: ghash-intel - Add PSHUFB macros
Add PSHUFB macros instead of repeating byte sequences, suggested by Ingo.
arch/x86/crypto/ghash-clmulni-intel_asm.S +6 -5
@@ -17,6 +17,7 @@ */ #include <linux/linkage.h> +#include <asm/i387.h> .align 16 .Lbswap_mask:
@@ -101,7 +102,7 @@ ENTRY(clmul_ghash_mul) movups (%rsi), SHASH movaps .Lbswap_mask, BSWAP # pshufb BSWAP, DATA - .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 + PSHUFB_XMM5_XMM0 call __clmul_gf128mul_ble # pshufb BSWAP, DATA .byte 0x66, 0x0f, 0x38, 0x00, 0xc5
@@ -119,12 +120,12 @@ ENTRY(clmul_ghash_update) movups (%rdi), DATA movups (%rcx), SHASH # pshufb BSWAP, DATA - .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 + PSHUFB_XMM5_XMM0 .align 4 .Lupdate_loop: movups (%rsi), IN1 # pshufb BSWAP, IN1 - .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 + PSHUFB_XMM5_XMM6 pxor IN1, DATA call __clmul_gf128mul_ble sub $16, %rdx
@@ -132,7 +133,7 @@ ENTRY(clmul_ghash_update) cmp $16, %rdx jge .Lupdate_loop # pshufb BSWAP, DATA - .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 + PSHUFB_XMM5_XMM0 movups DATA, (%rdi) .Lupdate_just_ret: ret
@@ -146,7 +147,7 @@ ENTRY(clmul_ghash_setkey) movaps .Lbswap_mask, BSWAP movups (%rsi), %xmm0 # pshufb BSWAP, %xmm0 - .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 + PSHUFB_XMM5_XMM0 movaps %xmm0, %xmm1 psllq $1, %xmm0 psrlq $63, %xmm1
arch/x86/include/asm/i387.h +7 -0
@@ -10,6 +10,8 @@#ifndef _ASM_X86_I387_H#define _ASM_X86_I387_H +#ifndef __ASSEMBLY__ +#include <linux/sched.h> #include <linux/kernel_stat.h> #include <linux/regset.h>
@@ -411,4 +413,9 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } +#endif /* __ASSEMBLY__ */ + +#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 +#define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5 +#endif /* _ASM_X86_I387_H */
Thanks,
Signature · 4 lines
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txtQuoted 14 lines · Ingo Molnar
Quoted 8 lines · Herbert Xu
Quoted 4 lines · Ingo Molnar
A cleanup request: mind creating two macros for this PSHUFB MMX/SSE instruction in
arch/x86/include/asm/i387.h, instead of open-coding the .byte sequences in ~6 places?I had a go at doing that, but it seems that
i387.hisn't really meant to be included in an asm file at this point :)Please use the standard construct and put an #ifndef
__ASSEMBLY__around it.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
( After the .33 merge window we'll collect such instruction format
knowledge inarch/x86/include/asm/inst.h. That file is not upstream yet soi387.hwill do for now for FPU/SSE instructions. )I'm happy to revisit this once
inst.hexists.No reason to not do most of the change first though, the way i suggested it.
How about something as below? But it seems not appropriate to put these bits into i387.h, that is, to combine C and gas syntax.
Best Regards,
Huang Ying
.macro xmm_num opd xmm
.ifc \xmm,%xmm0
\opd = 0
.endif
.ifc \xmm,%xmm1
\opd = 1
.endif
.ifc \xmm,%xmm2
\opd = 2
.endif
.ifc \xmm,%xmm3
\opd = 3
.endif
.ifc \xmm,%xmm4
\opd = 4
.endif
.ifc \xmm,%xmm5
\opd = 5
.endif
.ifc \xmm,%xmm6
\opd = 6
.endif
.ifc \xmm,%xmm7
\opd = 7
.endif
.ifc \xmm,%xmm8
\opd = 8
.endif
.ifc \xmm,%xmm9
\opd = 9
.endif
.ifc \xmm,%xmm10
\opd = 10
.endif
.ifc \xmm,%xmm11
\opd = 11
.endif
.ifc \xmm,%xmm12
\opd = 12
.endif
.ifc \xmm,%xmm13
\opd = 13
.endif
.ifc \xmm,%xmm14
\opd = 14
.endif
.ifc \xmm,%xmm15
\opd = 15
.endif
.endm
.macro PSHUFB_XMM xmm1 xmm2xmm_num pshufb_opd1 \xmm1xmm_num pshufb_opd2 \xmm2
.if (pshufb_opd1 < 8) && (pshufb_opd2 < 8)
.byte 0x66, 0x0f, 0x38, 0x00, 0xc0 | pshufb_opd1 | (pshufb_opd2<<3) .elseif (pshufb_opd1 >= 8) && (pshufb_opd2 < 8)
.byte 0x66, 0x41, 0x0f, 0x38, 0x00, 0xc0 | (pshufb_opd1-8) | (pshufb_opd2<<3) .elseif (pshufb_opd1 < 8) && (pshufb_opd2 >= 8)
.byte 0x66, 0x44, 0x0f, 0x38, 0x00, 0xc0 | pshufb_opd1 | ((pshufb_opd2-8)<<3) .else
.byte 0x66, 0x45, 0x0f, 0x38, 0x00, 0xc0 | (pshufb_opd1-8) | ((pshufb_opd2-8)<<3) .endif
.endm
Ingo Molnar
Acked-byQuoted 81 lines · Huang Ying
Quoted 14 lines · Ingo Molnar
Quoted 8 lines · Herbert Xu
Quoted 4 lines · Ingo Molnar
A cleanup request: mind creating two macros for this PSHUFB MMX/SSE instruction in
arch/x86/include/asm/i387.h, instead of open-coding the .byte sequences in ~6 places?I had a go at doing that, but it seems that
i387.hisn't really meant to be included in an asm file at this point :)Please use the standard construct and put an #ifndef
__ASSEMBLY__around it.Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
( After the .33 merge window we'll collect such instruction format
knowledge inarch/x86/include/asm/inst.h. That file is not upstream yet soi387.hwill do for now for FPU/SSE instructions. )I'm happy to revisit this once
inst.hexists.No reason to not do most of the change first though, the way i suggested it.
How about something as below? But it seems not appropriate to put these bits into
i387.h, that is, to combine C and gas syntax.Best Regards,
Huang Ying.macro
xmm_numopd xmm
.ifc \xmm,%xmm0
\opd = 0
.endif
.ifc \xmm,%xmm1
\opd = 1
.endif
.ifc \xmm,%xmm2
\opd = 2
.endif
.ifc \xmm,%xmm3
\opd = 3
.endif
.ifc \xmm,%xmm4
\opd = 4
.endif
.ifc \xmm,%xmm5
\opd = 5
.endif
.ifc \xmm,%xmm6
\opd = 6
.endif
.ifc \xmm,%xmm7
\opd = 7
.endif
.ifc \xmm,%xmm8
\opd = 8
.endif
.ifc \xmm,%xmm9
\opd = 9
.endif
.ifc \xmm,%xmm10
\opd = 10
.endif
.ifc \xmm,%xmm11
\opd = 11
.endif
.ifc \xmm,%xmm12
\opd = 12
.endif
.ifc \xmm,%xmm13
\opd = 13
.endif
.ifc \xmm,%xmm14
\opd = 14
.endif
.ifc \xmm,%xmm15
\opd = 15
.endif
.endm
Quoted 13 lines
.macro
PSHUFB_XMMxmm1 xmm2xmm_numpshufb_opd1\xmm1xmm_numpshufb_opd2\xmm2
.if (pshufb_opd1< 8) && (pshufb_opd2< 8)
.byte0x66,0x0f,0x38,0x00,0xc0|pshufb_opd1| (pshufb_opd2<<3) .elseif (pshufb_opd1>= 8) && (pshufb_opd2< 8)
.byte0x66,0x41,0x0f,0x38,0x00,0xc0| (pshufb_opd1-8) | (pshufb_opd2<<3) .elseif (pshufb_opd1< 8) && (pshufb_opd2>= 8)
.byte0x66,0x44,0x0f,0x38,0x00,0xc0|pshufb_opd1| ((pshufb_opd2-8)<<3) .else
.byte0x66,0x45,0x0f,0x38,0x00,0xc0| (pshufb_opd1-8) | ((pshufb_opd2-8)<<3) .endif
.endm
Looks far too clever, i like it :-) We have quite a few assembly macros in arch/x86/include/asm/. The above one could be put into calling.h for example.
But the simpler .byte solution in i387.h would be fine too.
If you guys want to put helper define into arch/x86/include/asm/ into the crypto tree, feel free:
it would be clumsy to keep it separately in the x86 tree. Just dont spread raw .byte sequences in .S files please ...
Ingo
H. Peter Anvin
reviewerQuoted 66 lines · Ingo Molnar
Quoted 52 lines
.macro
xmm_numopd xmm
.ifc \xmm,%xmm0
\opd = 0
.endif
.ifc \xmm,%xmm1
\opd = 1
.endif
.ifc \xmm,%xmm2
\opd = 2
.endif
.ifc \xmm,%xmm3
\opd = 3
.endif
.ifc \xmm,%xmm4
\opd = 4
.endif
.ifc \xmm,%xmm5
\opd = 5
.endif
.ifc \xmm,%xmm6
\opd = 6
.endif
.ifc \xmm,%xmm7
\opd = 7
.endif
.ifc \xmm,%xmm8
\opd = 8
.endif
.ifc \xmm,%xmm9
\opd = 9
.endif
.ifc \xmm,%xmm10
\opd = 10
.endif
.ifc \xmm,%xmm11
\opd = 11
.endif
.ifc \xmm,%xmm12
\opd = 12
.endif
.ifc \xmm,%xmm13
\opd = 13
.endif
.ifc \xmm,%xmm14
\opd = 14
.endif
.ifc \xmm,%xmm15
\opd = 15
.endif
.endmQuoted 13 lines
.macro
PSHUFB_XMMxmm1 xmm2xmm_numpshufb_opd1\xmm1xmm_numpshufb_opd2\xmm2
.if (pshufb_opd1< 8) && (pshufb_opd2< 8)
.byte0x66,0x0f,0x38,0x00,0xc0|pshufb_opd1| (pshufb_opd2<<3) .elseif (pshufb_opd1>= 8) && (pshufb_opd2< 8)
.byte0x66,0x41,0x0f,0x38,0x00,0xc0| (pshufb_opd1-8) | (pshufb_opd2<<3) .elseif (pshufb_opd1< 8) && (pshufb_opd2>= 8)
.byte0x66,0x44,0x0f,0x38,0x00,0xc0|pshufb_opd1| ((pshufb_opd2-8)<<3) .else
.byte0x66,0x45,0x0f,0x38,0x00,0xc0| (pshufb_opd1-8) | ((pshufb_opd2-8)<<3) .endif
.endm
Quoted 4 lines
Looks far too clever, i like it :-) We have quite a few assembly macros in
arch/x86/include/asm/. The above one could be put intocalling.hfor example.
I would really like to see something like that, with only one minor
tweak: please use submacros to generate the REX and MODRM bytes, since we are *guaranteed* to want to do the same thing again.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.