Skip to contentSkip to conversation

Add PowerPC chacha20 implementation from openssl/cryptograms

PATCH New Shawn Landden 1 participant 5 messages May 11, 2019 – May 18, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Add PowerPC chacha20 implementation from openssl/cryptograms

There is a bug where may_use_simd()
returns false in kworker, and prevents most of this code from running.

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le). It is quite a bit faster (look at the benchmarks in the assembly file) but it is difficult to give numbers because the interface line speed is the limiting factor.

---
 src/compat/simd/include/linux/simd.h         |   18 +-
 src/crypto/Kbuild.include                    |   26 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   67 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1818 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl
src/compat/simd/include/linux/simd.h +16 -2
index c75c724..23af478 100644
@@ -11,10 +11,12 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +30,23 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_VSX)
+		disable_kernel_vsx();
+		preempt_enable();
+#elif defined(CONFIG_ALTIVEC)
+		disable_kernel_altivec();
+		preempt_enable();
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +68,16 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_VSX)
+	preempt_disable();
+	enable_kernel_vsx();
+#elif defined(CONFIG_ALTIVEC)
+	preempt_disable();
+	enable_kernel_altivec();
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
src/crypto/Kbuild.include +25 -1
index 460684d..64894a5 100644
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,38 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
+ifdef ($(CONFIG_VSX),y)
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_VSX
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_VSX
+endif
+ifdef ($(CONFIG_ALTIVEC),y)
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ALTIVEC
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ALTIVEC
+endif
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
src/crypto/zinc/chacha20/chacha20-ppc-glue.c new file +67 -0

67 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20-ppc.pl new file +1355 -0

1,355 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20.c +2 -0
index b4763c8..42e5360 100644
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
src/crypto/zinc/chacha20/ppc-xlate.pl new file +353 -0

353 lines not on this page · raw message

Signature · 6 lines
2.20.1

_______________________________________________
WireGuard mailing list
[email protected]
https://lists.zx2c4.com/mailman/listinfo/wireguard
#1 lore raw

PATCH 1/2 Add PowerPC chacha20 implementation from openssl/cryptograms

There is a bug where may_use_simd()
returns false in kworker, and prevents most of this code from running. I reported this upstream: https://bugzilla.kernel.org/show_bug.cgi?id=203571

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le). Without this I get 2GiB/s over the loopback (so 4 GiB/s), and with this I get 2.8 GiB/s (so 5.6 GiB/s), and more time is spent in poly1305 than chacha20. This is on a 4-thread VPS.

v2: more complete simd.h for PPC
benchmarks
whitespace issues

---
 src/compat/simd/include/linux/simd.h         |   29 +-
 src/crypto/Kbuild.include                    |   18 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   67 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1821 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl
src/compat/simd/include/linux/simd.h +27 -2
index c75c724..44060a9 100644
@@ -11,10 +11,13 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
+#include <asm/cputable.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +31,28 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			disable_kernel_vsx();
+			preempt_enable();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			disable_kernel_altivec();
+			preempt_enable();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			disable_kernel_fp();
+			preempt_enable();
+		}
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +74,21 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			preempt_disable();
+			enable_kernel_vsx();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			preempt_disable();
+			enable_kernel_altivec();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			preempt_disable();
+			enable_kernel_fp();
+		}
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
src/crypto/Kbuild.include +17 -1
index 460684d..4e05181 100644
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,30 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
src/crypto/zinc/chacha20/chacha20-ppc-glue.c new file +67 -0

67 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20-ppc.pl new file +1355 -0

1,355 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20.c +2 -0
index b4763c8..42e5360 100644
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
src/crypto/zinc/chacha20/ppc-xlate.pl new file +353 -0

353 lines not on this page · raw message

Signature · 6 lines
2.20.1

_______________________________________________
WireGuard mailing list
[email protected]
https://lists.zx2c4.com/mailman/listinfo/wireguard
#2 lore raw

PATCH 2/2 add accelerated poly1305 from openssl/cryptograms

Unfortunately I am not seeing a speed up with this patch, but it does decrease CPU usage.

Again, this will not work until ppc has its own version of the may_use_simd() function that allows simd in the softirq context that WireGuard runs in.

---
 src/crypto/Kbuild.include                     |   10 +-
 src/crypto/zinc/chacha20/chacha20-ppc.pl      |    3 +
 .../zinc/{chacha20 => perlasm}/ppc-xlate.pl   |    0
 src/crypto/zinc/poly1305/poly1305-arm-glue.c  |   65 -
 src/crypto/zinc/poly1305/poly1305-ppc-glue.c  |   94 +
 src/crypto/zinc/poly1305/poly1305-ppc.pl      | 1989 +++++++++++++++++
 src/crypto/zinc/poly1305/poly1305-ppcfp.pl    |  749 +++++++
 src/crypto/zinc/poly1305/poly1305.c           |   69 +
 8 files changed, 2910 insertions(+), 69 deletions(-)
 rename src/crypto/zinc/{chacha20 => perlasm}/ppc-xlate.pl (100%)
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc-glue.c
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc.pl
 create mode 100755 src/crypto/zinc/poly1305/poly1305-ppcfp.pl
src/crypto/Kbuild.include +6 -4
index 4e05181..ea18ce5 100644
@@ -34,30 +34,32 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o
 AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots
 zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += poly1305/poly1305-ppc.o poly1305/poly1305-ppcfp.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += poly1305/poly1305-ppc.o poly1305/poly1305-ppcfp.o
 
 zinc-y += chacha20poly1305.o
 
 zinc-y += blake2s/blake2s.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
+      cmd_perlasm = $(PERL) $(perlflags-y) $< $(perlargs-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
-perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+perlargs-$(CONFIG_ZINC_ARCH_PPC32) += linux32
 ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
-perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
-perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+perlargs-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlargs-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
 endif
 
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
src/crypto/zinc/chacha20/chacha20-ppc.pl +3 -0
index 07468c8..fa8f6bc 100644
@@ -4,10 +4,12 @@
 # This code is taken from the OpenSSL project but the author, Andy Polyakov,
 # has relicensed it under the licenses specified in the SPDX header above.
 # The original headers, including the original license headers, are
 # included below for completeness.
 #
+# Changes: search in more places for ppc-xlate.pl
+#
 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
@@ -71,10 +73,11 @@ if ($flavour =~ /64/) {
 
 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
 
 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
 die "can't locate ppc-xlate.pl";
 
 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
 
src/crypto/zinc/perlasm/ppc-xlate.pl renamed from src/crypto/zinc/chacha20/ppc-xlate.pl +0 -0
similarity index 100%
rename from src/crypto/zinc/chacha20/ppc-xlate.pl
rename to src/crypto/zinc/perlasm/ppc-xlate.pl
src/crypto/zinc/poly1305/poly1305-arm-glue.c +0 -65

75 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305-ppc-glue.c new file +94 -0

94 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305-ppc.pl new file +1989 -0

1,989 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305-ppcfp.pl new file +749 -0

749 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305.c +69 -0

34 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 6 lines
2.20.1

_______________________________________________
WireGuard mailing list
[email protected]
https://lists.zx2c4.com/mailman/listinfo/wireguard

Part of the diff is not shown here. View the raw message.

#3 lore raw

PATCH v3 1/2 Add PowerPC chacha20 implementation from openssl/cryptograms

In reply to Shawn Landden

Only runs on the out-bound path, as the in-bound path is in an interrupt, but that can be fixed in Linux.

Otherwise this is tested (with the VSX code enabled) on Power 9 (ppc64le). Without this I get 2GiB/s over the loopback (so 4 GiB/s), and with this I get 2.8 GiB/s (so 5.6 GiB/s), and more time is spent in poly1305 than chacha20. This is on a 4-thread VPS.

v2: more complete simd.h for PPC
benchmarks
whitespace issues
v3: honor CONFIG_ALTIVEC (CONFIG_VSX has already honored)

---
 src/compat/simd/include/linux/simd.h         |   29 +-
 src/crypto/Kbuild.include                    |   18 +-
 src/crypto/zinc/chacha20/chacha20-ppc-glue.c |   66 +
 src/crypto/zinc/chacha20/chacha20-ppc.pl     | 1355 ++++++++++++++++++
 src/crypto/zinc/chacha20/chacha20.c          |    2 +
 src/crypto/zinc/chacha20/ppc-xlate.pl        |  353 +++++
 6 files changed, 1820 insertions(+), 3 deletions(-)
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc-glue.c
 create mode 100644 src/crypto/zinc/chacha20/chacha20-ppc.pl
 create mode 100644 src/crypto/zinc/chacha20/ppc-xlate.pl
src/compat/simd/include/linux/simd.h +27 -2
index c75c724..44060a9 100644
@@ -11,10 +11,13 @@
 #if defined(CONFIG_X86_64)
 #include <linux/version.h>
 #include <asm/fpu/api.h>
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 #include <asm/neon.h>
+#elif defined(CONFIG_ALTIVEC) || defined(CONFIG_VSX)
+#include <asm/switch_to.h>
+#include <asm/cputable.h>
 #endif
 
 typedef enum {
 	HAVE_NO_SIMD = 1 << 0,
 	HAVE_FULL_SIMD = 1 << 1,
@@ -28,17 +31,28 @@ static inline void simd_get(simd_context_t *ctx)
 	*ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
 }
 
 static inline void simd_put(simd_context_t *ctx)
 {
+        if (*ctx & HAVE_SIMD_IN_USE) {
 #if defined(CONFIG_X86_64)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_fpu_end();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
-	if (*ctx & HAVE_SIMD_IN_USE)
 		kernel_neon_end();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			disable_kernel_vsx();
+			preempt_enable();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			disable_kernel_altivec();
+			preempt_enable();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			disable_kernel_fp();
+			preempt_enable();
+		}
 #endif
+	}
 	*ctx = HAVE_NO_SIMD;
 }
 
 static inline bool simd_relax(simd_context_t *ctx)
 {
@@ -60,10 +74,21 @@ static __must_check inline bool simd_use(simd_context_t *ctx)
 		return true;
 #if defined(CONFIG_X86_64)
 	kernel_fpu_begin();
 #elif defined(CONFIG_KERNEL_MODE_NEON)
 	kernel_neon_begin();
+#elif defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+		if (cpu_has_feature(CPU_FTR_VSX_COMP)) {
+			preempt_disable();
+			enable_kernel_vsx();
+		} else if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP)) {
+			preempt_disable();
+			enable_kernel_altivec();
+		} else if (!cpu_has_feature(CPU_FTR_FPU_UNAVAILABLE)) {
+			preempt_disable();
+			enable_kernel_fp();
+		}
 #endif
 	*ctx |= HAVE_SIMD_IN_USE;
 	return true;
 }
 
src/crypto/Kbuild.include +17 -1
index 460684d..4e05181 100644
@@ -11,17 +11,25 @@ ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy)
 CONFIG_ZINC_ARCH_MIPS := y
 endif
 ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy)
 CONFIG_ZINC_ARCH_MIPS64 := y
 endif
+ifeq ($(CONFIG_PPC32),y)
+CONFIG_ZINC_ARCH_PPC32 := y
+endif
+ifeq ($(CONFIG_PPC64),y)
+CONFIG_ZINC_ARCH_PPC64 := y
+endif
 
 zinc-y += chacha20/chacha20.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o
 AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += chacha20/chacha20-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += chacha20/chacha20-ppc.o
 
 zinc-y += poly1305/poly1305.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
@@ -36,22 +44,30 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< > $@
+      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
+perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
+perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+endif
+
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
 wireguard-y += $(addprefix crypto/zinc/,$(zinc-y))
 ccflags-y += -I$(src)/crypto/include
 ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64
 ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM
 ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64
 ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS
 ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64
+ccflags-$(CONFIG_ZINC_ARCH_PPC32) += -DCONFIG_ZINC_ARCH_PPC32
+ccflags-$(CONFIG_ZINC_ARCH_PPC64) += -DCONFIG_ZINC_ARCH_PPC64
 ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST
src/crypto/zinc/chacha20/chacha20-ppc-glue.c new file +66 -0

66 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20-ppc.pl new file +1355 -0

1,355 lines not on this page · raw message

src/crypto/zinc/chacha20/chacha20.c +2 -0
index b4763c8..42e5360 100644
@@ -20,10 +20,12 @@
 #include "chacha20-x86_64-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64)
 #include "chacha20-arm-glue.c"
 #elif defined(CONFIG_ZINC_ARCH_MIPS)
 #include "chacha20-mips-glue.c"
+#elif defined(CONFIG_ZINC_ARCH_PPC32) || defined(CONFIG_ZINC_ARCH_PPC64)
+#include "chacha20-ppc-glue.c"
 #else
 static bool *const chacha20_nobs[] __initconst = { };
 static void __init chacha20_fpu_init(void)
 {
 }
src/crypto/zinc/chacha20/ppc-xlate.pl new file +353 -0

353 lines not on this page · raw message

Signature · 6 lines
2.20.1

_______________________________________________
WireGuard mailing list
[email protected]
https://lists.zx2c4.com/mailman/listinfo/wireguard
#4 lore raw

PATCH v3 2/2 Add PowerPC accelerated poly1305 from openssl/cryptograms

Unfortunately I am not seeing a speed up with this patch, but it does decrease CPU usage.

Only (currently) runs on the outbound path, as the in-bound path is in an interrupt, but that can be fixed in Linux.

v2: - Do not include the FPU version, as +10% performance on POWER8
(admittedly better on really old CPUs, like old world macs) is not worth it, especially when there is a fast VSX version available.
- Honor CONFIG_VSX.

---
 src/crypto/Kbuild.include                     |   10 +-
 src/crypto/zinc/chacha20/chacha20-ppc.pl      |    3 +
 .../zinc/{chacha20 => perlasm}/ppc-xlate.pl   |    0
 src/crypto/zinc/poly1305/poly1305-arm-glue.c  |   65 -
 src/crypto/zinc/poly1305/poly1305-ppc-glue.c  |   60 +
 src/crypto/zinc/poly1305/poly1305-ppc.pl      | 1989 +++++++++++++++++
 src/crypto/zinc/poly1305/poly1305.c           |   69 +
 7 files changed, 2127 insertions(+), 69 deletions(-)
 rename src/crypto/zinc/{chacha20 => perlasm}/ppc-xlate.pl (100%)
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc-glue.c
 create mode 100644 src/crypto/zinc/poly1305/poly1305-ppc.pl
src/crypto/Kbuild.include +6 -4
index 4e05181..ece20c9 100644
@@ -34,30 +34,32 @@ zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o
 zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o
 zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o
 AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots
 zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o
+zinc-$(CONFIG_ZINC_ARCH_PPC32) += poly1305/poly1305-ppc.o
+zinc-$(CONFIG_ZINC_ARCH_PPC64) += poly1305/poly1305-ppc.o
 
 zinc-y += chacha20poly1305.o
 
 zinc-y += blake2s/blake2s.o
 zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o
 
 zinc-y += curve25519/curve25519.o
 zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o
 
 quiet_cmd_perlasm = PERLASM $@
-      cmd_perlasm = $(PERL) $< $(perlflags-y) > $@
+      cmd_perlasm = $(PERL) $(perlflags-y) $< $(perlargs-y) > $@
 $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perlasm)
 kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src))
 targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-))))
 
-perlflags-$(CONFIG_ZINC_ARCH_PPC32) += linux32
+perlargs-$(CONFIG_ZINC_ARCH_PPC32) += linux32
 ifeq ($(CONFIG_ZINC_ARCH_PPC64),y)
-perlflags-$(CONFIG_CPU_BIG_ENDIAN) += linux64
-perlflags-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
+perlargs-$(CONFIG_CPU_BIG_ENDIAN) += linux64
+perlargs-$(CONFIG_CPU_LITTLE_ENDIAN) += linux64le
 endif
 
 # Old kernels don't set this, which causes trouble.
 .SECONDARY:
 
src/crypto/zinc/chacha20/chacha20-ppc.pl +3 -0
index 07468c8..fa8f6bc 100644
@@ -4,10 +4,12 @@
 # This code is taken from the OpenSSL project but the author, Andy Polyakov,
 # has relicensed it under the licenses specified in the SPDX header above.
 # The original headers, including the original license headers, are
 # included below for completeness.
 #
+# Changes: search in more places for ppc-xlate.pl
+#
 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
@@ -71,10 +73,11 @@ if ($flavour =~ /64/) {
 
 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
 
 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../perlasm/ppc-xlate.pl" and -f $xlate) or
 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
 die "can't locate ppc-xlate.pl";
 
 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
 
src/crypto/zinc/perlasm/ppc-xlate.pl renamed from src/crypto/zinc/chacha20/ppc-xlate.pl +0 -0
similarity index 100%
rename from src/crypto/zinc/chacha20/ppc-xlate.pl
rename to src/crypto/zinc/perlasm/ppc-xlate.pl
src/crypto/zinc/poly1305/poly1305-arm-glue.c +0 -65

75 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305-ppc-glue.c new file +60 -0

60 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305-ppc.pl new file +1989 -0

1,989 lines not on this page · raw message

src/crypto/zinc/poly1305/poly1305.c +69 -0

85 lines not on this page · raw message

Signature · 6 lines
2.20.1

_______________________________________________
WireGuard mailing list
[email protected]
https://lists.zx2c4.com/mailman/listinfo/wireguard