Add PowerPC chacha20 implementation from openssl/cryptograms
PATCH RESEND v3 1/2 Add PowerPC chacha20 implementation from openssl/cryptograms
This is tested (with the VSX code enabled) on Power 9 (ppc64le). Without this I get 2GiB/s over the loopback (so 4 GiB/s), and with this I get 2.8 GiB/s (so 5.6 GiB/s), and more time is spent in poly1305 than chacha20. This is on a 4-thread VPS.
v2: more complete simd.h for PPC
benchmarks
whitespace issues
v3: honor CONFIG_ALTIVEC (CONFIG_VSX has already honored)
--- src/compat/simd/include/linux/simd.h | 29 +- src/crypto/Kbuild.include | 18 +- src/crypto/zinc/chacha20/chacha20-ppc-glue.c | 66 + src/crypto/zinc/chacha20/chacha20-ppc.pl | 1355 ++++++++++++++++++ src/crypto/zinc/chacha20/chacha20.c | 2 + src/crypto/zinc/chacha20/ppc-xlate.pl | 353 +++++ 6 files changed, 1820 insertions(+), 3 deletions(-) create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl
src/compat/simd/include/linux/simd.h +27 -2
@@ -11,10 +11,13 @@#if defined(CONFIG_X86_64 )#include <linux/version.h> #include <asm/fpu/api.h> #elif defined(CONFIG_KERNEL_MODE_NEON )#include <asm/neon.h> +#elif defined(CONFIG_ALTIVEC ) || defined(CONFIG_VSX ) +#include <asm/switch_to.h> +#include <asm/cputable.h> #endif typedef enum { HAVE_NO_SIMD = 1 << 0, HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +31,28 @@ static inline void simd_get(simd_context_t *ctx) *ctx = !IS_ENABLED (CONFIG_PREEMPT_RT_BASE ) &&may_use_simd () ?HAVE_FULL_SIMD :HAVE_NO_SIMD ; }static inline void simd_put (simd_context_t *ctx) { +if (*ctx &HAVE_SIMD_IN_USE ) {#if defined(CONFIG_X86_64 ) -if (*ctx &HAVE_SIMD_IN_USE )kernel_fpu_end ();#elif defined(CONFIG_KERNEL_MODE_NEON ) -if (*ctx &HAVE_SIMD_IN_USE )kernel_neon_end (); +#elif defined(CONFIG_PPC32 ) || defined(CONFIG_PPC64 ) +if (cpu_has_feature (CPU_FTR_VSX_COMP )) { +disable_kernel_vsx (); +preempt_enable (); + }else if (cpu_has_feature (CPU_FTR_ALTIVEC_COMP )) { +disable_kernel_altivec (); +preempt_enable (); + }else if (!cpu_has_feature (CPU_FTR_FPU_UNAVAILABLE )) { +disable_kernel_fp (); +preempt_enable (); + }#endif + } *ctx =HAVE_NO_SIMD ; }static inline bool simd_relax (simd_context_t *ctx) {
@@ -60,10 +74,21 @@ static __must_check inline bool simd_use(simd_context_t *ctx)return true;#if defined(CONFIG_X86_64 )kernel_fpu_begin ();#elif defined(CONFIG_KERNEL_MODE_NEON )kernel_neon_begin (); +#elif defined(CONFIG_PPC32 ) || defined(CONFIG_PPC64 ) +if (cpu_has_feature (CPU_FTR_VSX_COMP )) { +preempt_disable (); +enable_kernel_vsx (); + }else if (cpu_has_feature (CPU_FTR_ALTIVEC_COMP )) { +preempt_disable (); +enable_kernel_altivec (); + }else if (!cpu_has_feature (CPU_FTR_FPU_UNAVAILABLE )) { +preempt_disable (); +enable_kernel_fp (); + }#endif *ctx |=HAVE_SIMD_IN_USE ;return true; }
src/crypto/Kbuild.include +17 -1
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy) CONFIG_ZINC_ARCH_MIPS := y endif ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy) CONFIG_ZINC_ARCH_MIPS64 := y endif +ifeq ($(CONFIG_PPC32),y) +CONFIG_ZINC_ARCH_PPC32 := y +endif +ifeq ($(CONFIG_PPC64),y) +CONFIG_ZINC_ARCH_PPC64 := y +endif zinc-y += chacha20/chacha20.o zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots +zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o +zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o zinc-y += poly1305/poly1305.o zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,30 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o zinc-y += curve25519/curve25519.o zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $< > $@ + cmd_perlasm = $(PERL) $< $(perlflags-y) > $@ $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perlasm) kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) +perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32 +ifeq ($(CONFIG_ZINC_ARCH_PPC64),y) +perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64 +perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le +endif + # Old kernels don't set this, which causes trouble. .SECONDARY: wireguard-y += $(addprefix crypto/zinc/,$(zinc-y)) ccflags-y += -I$(src)/crypto/include ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64 +ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32 +ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
src/crypto/zinc/chacha20/chacha20-ppc-glue.c new file +66 -0
66 lines not on this page · raw message
src/crypto/zinc/chacha20/chacha20-ppc.pl new file +1355 -0
1,355 lines not on this page · raw message
src/crypto/zinc/chacha20/chacha20.c +2 -0
@@ -20,10 +20,12 @@#include "chacha20-x86_64-glue.c" #elif defined(CONFIG_ZINC_ARCH_ARM ) || defined(CONFIG_ZINC_ARCH_ARM64 )#include "chacha20-arm-glue.c" #elif defined(CONFIG_ZINC_ARCH_MIPS )#include "chacha20-mips-glue.c" +#elif defined(CONFIG_ZINC_ARCH_PPC32 ) || defined(CONFIG_ZINC_ARCH_PPC64 ) +#include "chacha20-ppc-glue.c" #else static bool *const chacha20_nobs[]__initconst = { };static void __init chacha20_fpu_init (void ) { }
src/crypto/zinc/chacha20/ppc-xlate.pl new file +353 -0
353 lines not on this page · raw message
Signature · 6 lines
2.20.1 _______________________________________________ WireGuard mailing list [email protected] https://lists.zx2c4.com/mailman/listinfo/wireguard
PATCH RESEND v3 2/2 Add PowerPC accelerated poly1305 from openssl/cryptograms
Unfortunately I am not seeing a speed up with this patch, but it does decrease CPU usage.
v2: - Do not include the FPU version, as +10% performance on POWER8
(admittedly better on really old CPUs, like old world macs) is not worth it, especially when there is a fast VSX version available.
- Honor CONFIG_VSX.
---
src/crypto/Kbuild.include | 10 +-
src/crypto/zinc/chacha20/chacha20-ppc.pl | 3 +
.../zinc/{chacha20 => perlasm}/ppc-xlate.pl | 0
src/crypto/zinc/poly1305/poly1305-arm-glue.c | 65 -
src/crypto/zinc/poly1305/poly1305-ppc-glue.c | 60 +
src/crypto/zinc/poly1305/poly1305-ppc.pl | 1989 +++++++++++++++++
src/crypto/zinc/poly1305/poly1305.c | 69 +
7 files changed, 2127 insertions(+), 69 deletions(-)
rename src/crypto/zinc/{chacha20 => perlasm}/ppc-xlate.pl (100%)
create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc-glue.c
create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc.plsrc/crypto/Kbuild.include +6 -4
@@ -34,30 +34,32 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o +zinc-$(CONFIG_ZINC_ARCH_PPC32) += poly1305/poly1305-ppc.o +zinc-$(CONFIG_ZINC_ARCH_PPC64) += poly1305/poly1305-ppc.o zinc-y += chacha20poly1305.o zinc-y += blake2s/blake2s.o zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o zinc-y += curve25519/curve25519.o zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $< $(perlflags-y) > $@ + cmd_perlasm = $(PERL) $(perlflags-y) $< $(perlargs-y) > $@ $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perlasm) kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) -perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32 +perlargs-$(CONFIG_ZINC_ARCH_PPC32) += linux32 ifeq ($(CONFIG_ZINC_ARCH_PPC64),y) -perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64 -perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le +perlargs-$(CONFIG_CPU_BIG_ENDIAN) += linux64 +perlargs-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le endif # Old kernels don't set this, which causes trouble. .SECONDARY:
src/crypto/zinc/chacha20/chacha20-ppc.pl +3 -0
@@ -4,10 +4,12 @@ # This code is taken from the OpenSSL project but the author, Andy Polyakov, # has relicensed it under the licenses specified in the SPDX header above. # The original headers, including the original license headers, are # included below for completeness. # +# Changes: search in more places for ppc-xlate.pl +# # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. # # Licensed under the Apache License 2.0 (the "License"). You may not use # this file except in compliance with the License. You can obtain a copy # in the file LICENSE in the source distribution or at
@@ -71,10 +73,11 @@ if ($flavour =~ /64/) { $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0; $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
src/crypto/zinc/perlasm/ppc-xlate.pl renamed from src/crypto/zinc/chacha20/ppc-xlate.pl +0 -0
src/crypto/zinc/poly1305/poly1305-arm-glue.c +0 -65
75 lines not on this page · raw message
src/crypto/zinc/poly1305/poly1305-ppc-glue.c new file +60 -0
60 lines not on this page · raw message
src/crypto/zinc/poly1305/poly1305-ppc.pl new file +1989 -0
1,989 lines not on this page · raw message
src/crypto/zinc/poly1305/poly1305.c +69 -0
85 lines not on this page · raw message
Signature · 6 lines
2.20.1 _______________________________________________ WireGuard mailing list [email protected] https://lists.zx2c4.com/mailman/listinfo/wireguard