Experiments with AES-AltiVec, part 2

Index: linux/crypto/aes-alti.c
===================================================================
@@ -0,0 +1,274 @@
+/*
+ * based on crypto/aes.c
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <asm/byteorder.h>
+#include <asm/system.h>
+#include <crypto/algapi.h>
+
+#include "aes-altivec.h"
+
+#define AES_MIN_KEY_SIZE	16
+#define AES_MAX_KEY_SIZE	32
+
+#define AES_BLOCK_SIZE		16
+
+/* max rounds is 14. Every round needs 1 vector as key (=4 ints or 16 bytes)
+ * The first slot is the given key
+ */
+
+#define MAX_AES_ROUNDS 15
+#define MAX_AES_KEYSIZE_INT (MAX_AES_ROUNDS *4)
+#define MAX_AES_KEYSIZE_BYTE (MAX_AES_KEYSIZE_INT *4)
+#define ENCRYPT 0
+#define DECRYPT 1
+
+struct aes_ctx {
+	unsigned char key_enc_ch[MAX_AES_KEYSIZE_BYTE] __attribute__ ((aligned (16)));
+	unsigned char key_dec_ch[MAX_AES_KEYSIZE_BYTE] __attribute__ ((aligned (16)));
+	unsigned int key_length;
+};
+
+static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+		       unsigned int key_len)
+{
+	struct aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	u32 *flags = &tfm->crt_flags;
+	u32 i;
+
+	switch (key_len) {
+	case 16:
+	case 24:
+	case 32:
+		break;
+
+	default:
+		 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		 return -EINVAL;
+	}
+
+	preempt_disable();
+	enable_kernel_altivec();
+
+	ctx->key_length = key_len;
+	i = expand_key(in_key, key_len/4 , ctx->key_enc_ch, ctx->key_dec_ch);
+
+	preempt_enable();
+	return i;
+}
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	const struct aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	preempt_disable();
+	enable_kernel_altivec();
+
+	aes_encrypt_altivec(in, out, ctx->key_enc_ch, ctx->key_length);
+
+	preempt_enable();
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+	const struct aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	preempt_disable();
+	enable_kernel_altivec();
+
+	aes_decrypt_altivec(in, out, ctx->key_dec_ch, ctx->key_length);
+
+	preempt_enable();
+}
+
+static struct crypto_alg aes_alg = {
+	.cra_name			=	"aes",
+	.cra_driver_name	=	"aes-altivec",
+	.cra_priority		=	123,
+	.cra_flags			=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct aes_ctx),
+	.cra_alignmask		=	15,
+	.cra_module			=	THIS_MODULE,
+	.cra_list			=	LIST_HEAD_INIT(aes_alg.cra_list),
+	.cra_u				=	{
+		.cipher = {
+			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
+			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
+			.cia_setkey	   		= 	aes_set_key,
+			.cia_encrypt	 	=	aes_encrypt,
+			.cia_decrypt	  	=	aes_decrypt
+		}
+	}
+};
+
+/*
+ * The fn pointer is the last parameter, the remaining parameter are in caller's order.
+ * That way the compiler must not reorder everything :)
+ */
+static int mode_aes_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		struct scatterlist *src, unsigned int nbytes,
+		int (*fn) (const unsigned char *in,
+			unsigned char *out, const unsigned char *kp, unsigned int key_len,
+			unsigned int len, unsigned char *iv_), unsigned int mode)
+{
+	struct aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	const unsigned char *kp;
+	int ret;
+	u8 *out, *in;
+
+	kp = mode == ENCRYPT ? ctx->key_enc_ch : ctx->key_dec_ch;
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	ret = blkcipher_walk_virt(desc, &walk);
+
+	preempt_disable();
+	enable_kernel_altivec();
+
+	while ((nbytes = walk.nbytes)) {
+		/* only use complete blocks */
+		unsigned int n = nbytes & ~(15);
+
+		out = walk.dst.virt.addr;
+		in = walk.src.virt.addr;
+
+		nbytes = fn(in, out, kp, ctx->key_length, n, walk.iv);
+
+		nbytes &= 15;
+		ret = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	preempt_enable();
+	return ret;
+}
+
+static int aes_encrypt_ecb(struct blkcipher_desc *desc,
+		struct scatterlist *dst, struct scatterlist *src,
+		unsigned int nbytes)
+{
+	return mode_aes_crypt(desc, dst, src, nbytes, aes_encrypt_ecb_altivec, ENCRYPT);
+}
+
+static int aes_decrypt_ecb(struct blkcipher_desc *desc,
+		struct scatterlist *dst, struct scatterlist *src,
+		unsigned int nbytes)
+{
+	return mode_aes_crypt(desc, dst, src, nbytes, aes_decrypt_ecb_altivec, DECRYPT);
+}
+
+static struct crypto_alg aes_ecb_alg = {
+	.cra_name           =   "ecb(aes)",
+	.cra_driver_name    =   "ecb-aes-altivec",
+	.cra_priority       =   125,
+	.cra_flags          =   CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize      =   AES_BLOCK_SIZE,
+	.cra_alignmask		=	15,
+	.cra_ctxsize        =   sizeof(struct aes_ctx),
+	.cra_type           =   &crypto_blkcipher_type,
+	.cra_module         =   THIS_MODULE,
+	.cra_list           =   LIST_HEAD_INIT(aes_ecb_alg.cra_list),
+	.cra_u              =   {
+		.blkcipher = {
+			.min_keysize    =   AES_MIN_KEY_SIZE,
+			.max_keysize    =   AES_MAX_KEY_SIZE,
+			.setkey         =   aes_set_key,
+			.encrypt        =   aes_encrypt_ecb,
+			.decrypt        =   aes_decrypt_ecb,
+		}
+	}
+};
+
+static int aes_encrypt_cbc(struct blkcipher_desc *desc,
+		struct scatterlist *dst, struct scatterlist *src,
+		unsigned int nbytes)
+{
+	return mode_aes_crypt(desc, dst, src, nbytes, aes_encrypt_cbc_altivec, ENCRYPT);
+}
+
+static int aes_decrypt_cbc(struct blkcipher_desc *desc,
+		struct scatterlist *dst, struct scatterlist *src,
+		unsigned int nbytes)
+{
+	return mode_aes_crypt(desc, dst, src, nbytes, aes_decrypt_cbc_altivec, DECRYPT);
+}
+
+static struct crypto_alg aes_cbc_alg = {
+	.cra_name           =   "cbc(aes)",
+	.cra_driver_name    =   "cbc-aes-altivec",
+	.cra_priority       =   125,
+	.cra_flags          =   CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize      =   AES_BLOCK_SIZE,
+	.cra_alignmask		=	15,
+	.cra_ctxsize        =   sizeof(struct aes_ctx),
+	.cra_type           =   &crypto_blkcipher_type,
+	.cra_module         =   THIS_MODULE,
+	.cra_list           =   LIST_HEAD_INIT(aes_ecb_alg.cra_list),
+	.cra_u              =   {
+		.blkcipher = {
+			.min_keysize    =   AES_MIN_KEY_SIZE,
+			.max_keysize    =   AES_MAX_KEY_SIZE,
+			.setkey         =   aes_set_key,
+			.encrypt        =   aes_encrypt_cbc,
+			.decrypt        =   aes_decrypt_cbc,
+			.ivsize         =   AES_BLOCK_SIZE,
+		}
+	}
+};
+
+static int __init aes_init(void)
+{
+	unsigned int ret;
+
+	if (!(cpu_has_feature(CPU_FTR_ALTIVEC))) {
+		printk("aes-alti: No altivec unit available\n");
+		return -ENODEV;
+	}
+#ifdef CONFIG_CRYPTO_AES_ALTIVEC_TABLE
+	printk("Table lookup mode\n");
+#endif
+
+	ret = crypto_register_alg(&aes_alg);
+	if (ret) {
+		printk("Failed to register aes\n");
+		goto failed_aes;
+	}
+
+	ret = crypto_register_alg(&aes_ecb_alg);
+	if (ret) {
+		printk("Failed to register aes-ecb\n");
+		goto failed_aes_ecb;
+	}
+
+	ret = crypto_register_alg(&aes_cbc_alg);
+	if (ret) {
+		printk("Failed to register aes-cbc\n");
+		goto failed_aes_cbc;
+	}
+
+	return 0;
+
+failed_aes_cbc:
+	crypto_unregister_alg(&aes_ecb_alg);
+
+failed_aes_ecb:
+	crypto_unregister_alg(&aes_alg);
+
+failed_aes:
+	return -ENODEV;
+}
+
+static void __exit aes_fini(void)
+{
+	crypto_unregister_alg(&aes_alg);
+	crypto_unregister_alg(&aes_ecb_alg);
+	crypto_unregister_alg(&aes_cbc_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
+MODULE_LICENSE("GPL");

799 lines not on this page · raw message

Index: linux/crypto/aes-altivec.h
===================================================================
@@ -0,0 +1,28 @@
+#ifndef  __AES_ALTIVEC_H__
+#define  __AES_ALTIVEC_H__
+
+extern int expand_key(const unsigned char *key, unsigned int keylen,
+		unsigned char exp_enc_key[15 *4*4], unsigned char expanded_dec_key[15*4*4]);
+
+extern int aes_encrypt_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len);
+
+extern int aes_decrypt_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len);
+
+extern int aes_encrypt_ecb_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len, unsigned int len,
+		unsigned char *iv_);
+
+extern int aes_decrypt_ecb_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len, unsigned int len,
+		unsigned char *iv_);
+
+extern int aes_encrypt_cbc_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len, unsigned int len,
+		unsigned char *iv_);
+
+extern int aes_decrypt_cbc_altivec(const unsigned char *in, unsigned char *out,
+		const unsigned char *kp, unsigned int key_len, unsigned int len,
+		unsigned char *iv_);
+#endif

Index: linux/arch/powerpc/kernel/process.c
===================================================================
@@ -119,15 +119,21 @@ int dump_task_fpu(struct task_struct *ts
 #ifdef CONFIG_ALTIVEC
 void enable_kernel_altivec(void)
 {
-	WARN_ON(preemptible());
+	BUG_ON(preemptible());
+	/*
+	 * enable_kernel_altivec() will just save current AltiVec registers (if needed) and
+	 * return to caller (with MSR_VEC unchanged (probably not set)). The first AltiVec
+	 * instruction will raise an exception and the exception will enable the AltiVec for
+	 * the kernel. This is done to avoid the expensive "enable altivec" operation if it
+	 * is allready enabled. However, you have to disable preemtion while you are using
+	 * AltiVec.
+	 */
 
 #ifdef CONFIG_SMP
 	if (current->thread.regs && (current->thread.regs->msr & MSR_VEC))
-		giveup_altivec(current);
-	else
-		giveup_altivec(NULL);	/* just enable AltiVec for kernel - force */
+		giveup_user_altivec(current);
 #else
-	giveup_altivec(last_task_used_altivec);
+	giveup_user_altivec(last_task_used_altivec);
 #endif /* CONFIG_SMP */
 }
 EXPORT_SYMBOL(enable_kernel_altivec);

Index: linux/arch/powerpc/kernel/traps.c
===================================================================
@@ -886,6 +886,12 @@ void altivec_unavailable_exception(struc
 	die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT);
 }
 
+void altivec_enable_for_kernel_exception(struct pt_regs *regs)
+{
+	printk("altivec_enable_for_kernel_exception: AltiVec mode on for kernel\n");
+	regs->msr |= MSR_VEC;
+}
+
 void performance_monitor_exception(struct pt_regs *regs)
 {
 	perf_irq(regs);

Index: linux/include/asm-powerpc/system.h
===================================================================
@@ -129,6 +129,7 @@ extern void enable_kernel_fp(void);
 extern void flush_fp_to_thread(struct task_struct *);
 extern void enable_kernel_altivec(void);
 extern void giveup_altivec(struct task_struct *);
+extern void giveup_user_altivec(struct task_struct *);
 extern void load_up_altivec(struct task_struct *);
 extern int emulate_altivec(struct pt_regs *);
 extern void giveup_spe(struct task_struct *);

Index: linux/crypto/limi-speed.c
===================================================================
@@ -0,0 +1,140 @@
+/*
+ * Code derived von crypt/tcrypt.h
+ *
+ * Small speed test with time resolution in msec.
+ * Author: Sebastian Siewior (bigeasy _at_ breakpoint.cc)
+ * License: GPL v2
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <linux/jiffies.h>
+#include <linux/types.h>
+
+static char *in;
+
+static unsigned int buff_size = 16 * 1024;
+module_param(buff_size, uint, 0444);
+MODULE_PARM_DESC(buff_size, "Buffer allocated by kmalloc()");
+
+static unsigned int keylen = 16;
+module_param(keylen, uint, 0444);
+MODULE_PARM_DESC(keylen, "Length of the key (16,24 or 32 bits");
+
+static unsigned int mode = 0;
+module_param(mode, uint, 0444);
+MODULE_PARM_DESC(mode, "0 -> encryption else decryption");
+
+static unsigned int big_loops = 10;
+module_param(big_loops, uint, 0444);
+MODULE_PARM_DESC(big_loops, "Number of mensurations.");
+
+static unsigned int small_loops = 10000;
+module_param(small_loops, uint, 0444);
+MODULE_PARM_DESC(small_loops, "loops within one mesurement.");
+
+static unsigned int alg = 1;
+module_param(alg, uint, 0444);
+MODULE_PARM_DESC(alg, "0 -> ecb(aes), else -> cbc(aes)");
+
+static int __init init(void)
+{
+	struct scatterlist sg[1];
+	struct crypto_blkcipher *tfm;
+	struct blkcipher_desc desc;
+	unsigned int i;
+	unsigned int ret;
+	unsigned int iv_len;
+	unsigned long start, end;
+	unsigned long total = 0;
+	unsigned long size_kb;
+    unsigned char key[32] = { 1, 2, 3, 4, 5, 6 };
+	const unsigned char *algname;
+
+	algname = alg ? "cbc(aes)" : "ecb(aes)";
+	printk("Limi-speed: %s buff_size: %u, keylen: %d, mode: %s\n", algname, buff_size, keylen,
+			mode ? "decryption" : "encryption");
+	printk("loops: %d, iterations: %d, ", big_loops, small_loops);
+	size_kb = small_loops * buff_size / 1024;
+	printk("=> %lu kb or %lu mb a loop\n", size_kb, size_kb/1024);
+
+	if (keylen != 16 && keylen != 24 && keylen != 32) {
+		printk("Invalid keysize\n");
+		return -EINVAL;
+	}
+
+	in = kmalloc(buff_size, GFP_KERNEL);
+	if (in == NULL) {
+		printk("Failed to allocate memory.\n");
+		return -ENOMEM;
+	}
+
+	memset(in, 0x24, buff_size);
+
+	sg_set_buf(sg, in, buff_size);
+
+	tfm = crypto_alloc_blkcipher(algname, 0, CRYPTO_ALG_ASYNC);
+
+	if (IS_ERR(tfm)) {
+		printk("failed to load transform for %s: %ld\n", algname, PTR_ERR(tfm));
+		goto leave;
+	}
+
+	crypto_blkcipher_setkey(tfm, key, keylen);
+
+	iv_len = crypto_blkcipher_ivsize(tfm);
+	if (iv_len)
+		crypto_blkcipher_set_iv(tfm, in, iv_len);
+
+	desc.tfm = tfm;
+	desc.flags = 0;
+
+	for (i=0 ; i<big_loops; i++) {
+		int j;
+		start = jiffies;
+		ret = 0;
+
+		for (j=0; j < small_loops && !ret; j++) {
+
+			if (!mode)
+				ret = crypto_blkcipher_encrypt(&desc, sg, sg, buff_size);
+			else
+				ret = crypto_blkcipher_decrypt(&desc, sg, sg, buff_size);
+		}
+
+		if (ret) {
+			printk("encryption failed: %d after (i,j) (%u,%u) iterations\n", ret, i, j);
+			goto leave_loop;
+		}
+		end = jiffies;
+		if ( !time_after(start, end)) {
+			printk("Run: %u msec\n", jiffies_to_msecs(end - start));
+			total += jiffies_to_msecs(end - start);
+		} else {
+			printk("Run: %u msec\n", jiffies_to_msecs(start - end));
+			total += jiffies_to_msecs(start - end);
+		}
+	}
+
+	total /= big_loops;
+	size_kb *= 1000;
+	size_kb /= total;
+	printk("Average: %lu msec, approx. %lu kb/sec || %lu mb/sec  \n", total,
+			size_kb, size_kb/1024);
+
+leave_loop:
+	crypto_free_blkcipher(tfm);
+
+leave:
+	kfree(in);
+	return -ENODEV;
+}
+
+static void __exit fini(void) { }
+
+module_init(init);
+module_exit(fini);
+
+MODULE_LICENSE("GPL");