Acrypto - asynchronous crypto layer for linux kernel 2.6

@@ -0,0 +1,245 @@
+/*
+ * 	acrypto.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __ACRYPTO_H
+#define __ACRYPTO_H
+
+#define SCACHE_NAMELEN		32
+
+struct crypto_session_initializer;
+struct crypto_data;
+typedef void (*crypto_callback_t) (struct crypto_session_initializer *,
+				   struct crypto_data *);
+
+struct crypto_device_stat 
+{
+	__u64 scompleted;
+	__u64 sfinished;
+	__u64 sstarted;
+	__u64 kmem_failed;
+	__u64 pool_failed;
+};
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/device.h>
+#include <linux/workqueue.h>
+#include <linux/mempool.h>
+
+#include <asm/scatterlist.h>
+
+#define DEBUG
+#ifdef DEBUG
+#define dprintk(f, a...) printk(f, ##a)
+#define dprintka(f, a...) printk(f, ##a)
+#else
+#define dprintk(f, a...)
+#define dprintka(f, a...)
+#endif
+
+extern void crypto_wake_lb(void);
+
+#define SESSION_COMPLETED	(1<<15)
+#define SESSION_FINISHED	(1<<14)
+#define SESSION_STARTED		(1<<13)
+#define SESSION_PROCESSED	(1<<12)
+#define SESSION_BINDED		(1<<11)
+#define SESSION_BROKEN		(1<<10)
+#define SESSION_FROM_CACHE	(1<<9)
+
+#define session_completed(s)	(s->ci.flags & SESSION_COMPLETED)
+#define complete_session(s)	do {s->ci.flags |= SESSION_COMPLETED;} while(0)
+#define uncomplete_session(s)	do {s->ci.flags &= ~SESSION_COMPLETED;} while (0)
+
+#define session_finished(s)	(s->ci.flags & SESSION_FINISHED)
+#define finish_session(s)	do {s->ci.flags |= SESSION_FINISHED;} while(0)
+#define unfinish_session(s)	do {s->ci.flags &= ~SESSION_FINISHED;} while (0)
+
+#define session_started(s)	(s->ci.flags & SESSION_STARTED)
+#define start_session(s)	do {s->ci.flags |= SESSION_STARTED;} while(0)
+#define unstart_session(s)	do {s->ci.flags &= ~SESSION_STARTED;} while (0)
+
+#define session_is_processed(s)		(s->ci.flags & SESSION_PROCESSED)
+#define start_process_session(s)	do {s->ci.flags |= SESSION_PROCESSED; s->ci.ptime = jiffies;} while(0)
+#define stop_process_session(s)		do {s->ci.flags &= ~SESSION_PROCESSED; s->ci.ptime = jiffies - s->ci.ptime; crypto_wake_lb();} while (0)
+
+#define session_binded(s)	(s->ci.flags & SESSION_BINDED)
+#define bind_session(s)		do {s->ci.flags |= SESSION_BINDED;} while(0)
+#define unbind_session(s)	do {s->ci.flags &= ~SESSION_BINDED;} while (0)
+#define sci_binded(ci)		(ci->flags & SESSION_BINDED)
+
+#define session_broken(s)	(s->ci.flags & SESSION_BROKEN)
+#define broke_session(s)	do {s->ci.flags |= SESSION_BROKEN;} while(0)
+#define unbroke_session(s)	do {s->ci.flags &= ~SESSION_BROKEN;} while (0)
+
+#define session_from_cache(s)		(s->ci.flags & SESSION_FROM_CACHE)
+#define mark_session_from_cache(s)	do {s->ci.flags |= SESSION_FROM_CACHE;} while(0)
+
+#define CRYPTO_MAX_PRIV_SIZE	1024
+
+#define DEVICE_BROKEN		(1<<0)
+
+#define device_broken(dev)	(dev->flags & DEVICE_BROKEN)
+#define broke_device(dev)	do {dev->flags |= DEVICE_BROKEN;} while(0)
+#define repair_device(dev)	do {dev->flags &= ~DEVICE_BROKEN;} while(0)
+
+struct crypto_capability {
+	u16 			operation;
+	u16 			type;
+	u16 			mode;
+	u16 			qlen;
+	u64 			ptime;
+	u64 			scomp;
+};
+
+struct crypto_session_initializer {
+	u16 			operation;
+	u16 			type;
+	u16 			mode;
+	u16 			priority;
+
+	u64 			id;
+	u64 			dev_id;
+
+	u32 			flags;
+
+	u32 			bdev;
+
+	u64 			ptime;
+
+	crypto_callback_t 	callback;
+};
+
+struct crypto_data {
+	struct			scatterlist *sg_src;
+	int 			sg_src_num;
+	struct			scatterlist *sg_dst;
+	int 			sg_dst_num;
+	struct			scatterlist *sg_key;
+	int 			sg_key_num;
+	struct			scatterlist *sg_iv;
+	int 			sg_iv_num;
+
+	void 			*priv;
+	unsigned int 		priv_size;
+};
+
+struct crypto_device {
+	char 			name[SCACHE_NAMELEN];
+
+	spinlock_t 		session_lock;
+	struct list_head 	session_list;
+
+	u64 			sid;
+	spinlock_t 		lock;
+
+	atomic_t 		refcnt;
+
+	u32 			flags;
+
+	u32 			id;
+
+	struct list_head 	cdev_entry;
+
+	void 			(*data_ready)(struct crypto_device *);
+
+	struct device_driver 	*driver;
+	struct device 		device;
+	struct class_device 	class_device;
+	struct completion 	dev_released;
+
+	spinlock_t 		stat_lock;
+	struct crypto_device_stat stat;
+
+	struct crypto_capability *cap;
+	int 			cap_number;
+
+	void 			*priv;
+
+	mempool_t		*session_pool;
+	kmem_cache_t		*session_cache;
+};
+
+struct crypto_route_head {
+	struct crypto_route 	*next;
+	struct crypto_route 	*prev;
+
+	__u32 			qlen;
+	spinlock_t 		lock;
+};
+
+struct crypto_route {
+	struct crypto_route 	*next;
+	struct crypto_route 	*prev;
+
+	struct crypto_route_head *list;
+	struct crypto_device 	*dev;
+
+	struct crypto_session_initializer ci;
+};
+
+struct crypto_session {
+	struct list_head 	dev_queue_entry;
+	struct list_head	main_queue_entry;
+
+	struct crypto_session_initializer ci;
+
+	struct crypto_data 	data;
+
+	spinlock_t 		lock;
+
+	struct work_struct 	work;
+
+	struct crypto_route_head route_list;
+
+	struct crypto_device 	*pool_dev;
+};
+
+struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *, struct crypto_data *);
+struct crypto_session *crypto_session_create(struct crypto_session_initializer *, struct crypto_data *);
+void crypto_session_destroy(struct crypto_session *);
+void crypto_session_add(struct crypto_session *);
+void crypto_session_dequeue_main(struct crypto_session *);
+void __crypto_session_dequeue_main(struct crypto_session *);
+void __crypto_session_dequeue_route(struct crypto_session *);
+void crypto_session_dequeue_route(struct crypto_session *);
+
+void crypto_device_get(struct crypto_device *);
+void crypto_device_put(struct crypto_device *);
+struct crypto_device *crypto_device_get_name(char *);
+
+int __crypto_device_add(struct crypto_device *);
+int crypto_device_add(struct crypto_device *);
+void __crypto_device_remove(struct crypto_device *);
+void crypto_device_remove(struct crypto_device *);
+int match_initializer(struct crypto_device *, struct crypto_session_initializer *);
+int __match_initializer(struct crypto_capability *, struct crypto_session_initializer *);
+
+void crypto_session_insert_main(struct crypto_device *dev, struct crypto_session *s);
+void crypto_session_insert(struct crypto_device *dev, struct crypto_session *s);
+void __crypto_session_insert(struct crypto_device *dev, struct crypto_session *s);
+
+#endif				/* __KERNEL__ */
+#endif				/* __ACRYPTO_H */

@@ -0,0 +1,322 @@
+/*
+ * 	async_provider.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/err.h>
+#include <linux/crypto.h>
+#include <linux/mm.h>
+#include <linux/blkdev.h>
+
+#include "acrypto.h"
+#include "crypto_stat.h"
+#include "crypto_def.h"
+#include "crypto_route.h"
+#include "crypto_user.h"
+
+static unsigned int trnum = 1;
+module_param(trnum, uint, 0);
+
+static void prov_data_ready(struct crypto_device *);
+
+static struct crypto_capability prov_caps[] = {
+		{CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 100},
+		{CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 100},
+		
+		{CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 100},
+		{CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 100},
+};
+static int prov_cap_number = sizeof(prov_caps)/sizeof(prov_caps[0]);
+
+static int need_exit;
+static char async_algo[] = "aes";
+static char async_key[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+
+static struct crypto_device pdev = {
+	.name			= "async_provider",
+	.data_ready		= prov_data_ready,
+	.cap			= &prov_caps[0],
+};
+
+static struct async_provider
+{
+	int 			num;
+	struct crypto_tfm 	*tfm;
+	wait_queue_head_t	async_wait_queue;
+	struct completion 	thread_exited;
+	struct crypto_device	pdev;
+} *aprov;
+
+static void prov_data_ready(struct crypto_device *dev)
+{
+	struct async_provider *p;
+
+	p = (struct async_provider *)dev->priv;
+
+	if (p)
+		wake_up_interruptible(&p->async_wait_queue);
+}
+
+static int async_thread(void *data)
+{
+	struct async_provider *p = (struct async_provider *)data;
+	struct crypto_device *dev = &p->pdev;
+	struct crypto_session *s, *n;
+	int i, err, keylen, ivlen;
+	u8 *key, *iv;
+	int zero_pnum_cnt = 0;
+
+	daemonize("%s", dev->name);
+	allow_signal(SIGTERM);
+
+	while (!need_exit) {
+		int num, pnum;
+		
+		if (need_exit)
+			break;
+
+		num = pnum = 0;
+		list_for_each_entry_safe(s, n, &dev->session_list, dev_queue_entry) {
+			num++;
+			
+			if (session_completed(s))
+				continue;
+
+			pnum++;
+
+			start_process_session(s);
+			
+			if (s->data.sg_src_num != s->data.sg_dst_num) {
+				dprintk("Broken session: sg_src_num [%d] != sg_dst_num [%d].\n",
+						s->data.sg_src_num, s->data.sg_dst_num);
+				broke_session(s);
+				goto out;
+			}
+			
+			/*
+			 * Simple case - key is small(it's size is less than PAGE_SIZE).
+			 * Assymetric crypto will require proper key sg handling.
+			 */
+			key = kmap(s->data.sg_key[0].page) + s->data.sg_key[0].offset;
+			keylen = s->data.sg_key[0].length;
+
+			err = crypto_cipher_setkey(p->tfm, key, keylen);
+			if (err) {
+				dprintk(KERN_ERR "Failed to set key [keylen=%d]: err=%d.\n", keylen, err);
+				broke_session(s);
+				goto out;
+			}
+
+			if (s->ci.mode != CRYPTO_MODE_ECB) {
+				if (!s->data.sg_iv || !s->data.sg_iv_num) {
+					dprintk("Crypto mode %d requires IV.\n", s->ci.mode);
+					broke_session(s);
+					goto out;
+				}
+				
+				iv = kmap(s->data.sg_iv[0].page) + s->data.sg_iv[0].offset;
+				ivlen = s->data.sg_iv[0].length;
+
+				if (!iv || !ivlen) {
+					dprintk("Crypto mode %d requires IV, whic is broken: iv=%p, ivlen=%d.\n", 
+							s->ci.mode, iv, ivlen);
+					broke_session(s);
+					goto out;
+				}
+
+				crypto_cipher_set_iv(p->tfm, iv, ivlen);
+			} else {
+				iv = NULL;
+				ivlen = 0;
+			}
+			
+			for (i=0; i<s->data.sg_src_num; ++i) {
+				u8 *dst, *src;
+				int len;
+				
+				dst = kmap_atomic(s->data.sg_dst[i].page, KM_USER0) + s->data.sg_dst[i].offset;
+				src = kmap_atomic(s->data.sg_src[i].page, KM_USER1) + s->data.sg_src[i].offset;
+				len = s->data.sg_src[i].length;
+
+				if (s->ci.operation == CRYPTO_OP_ENCRYPT)
+					err = crypto_cipher_encrypt(p->tfm, &s->data.sg_dst[i], &s->data.sg_src[i], s->data.sg_src[i].length);
+				else
+					err = crypto_cipher_decrypt(p->tfm, &s->data.sg_dst[i], &s->data.sg_src[i], s->data.sg_src[i].length);
+
+				kunmap_atomic(src, KM_USER1);
+				kunmap_atomic(dst, KM_USER0);
+			
+				s->data.sg_dst[i].length = s->data.sg_src[i].length;
+				s->data.sg_dst[i].offset = s->data.sg_src[i].offset;
+
+				if (err < 0) {
+					broke_session(s);
+					printk("operation=%02x, size=%u, err=%d.\n", s->ci.operation, s->data.sg_src[i].length, err);
+				}
+			}
+
+			kunmap(s->data.sg_key[0].page);
+
+			if (iv)
+				kunmap(s->data.sg_iv[0].page);
+			
+			dprintk("%lu: Completing session %llu [%llu] in %s.\n", 
+					jiffies, s->ci.id, s->ci.dev_id, pdev.name);
+out:
+			crypto_stat_complete_inc(s);
+			crypto_session_dequeue_route(s);
+			complete_session(s);
+			stop_process_session(s);
+		}
+
+		if (!pnum)
+			zero_pnum_cnt++;
+		else
+			zero_pnum_cnt = 0;
+
+		if (unlikely(zero_pnum_cnt == 1000)) {
+			zero_pnum_cnt = 0;
+			interruptible_sleep_on_timeout(&p->async_wait_queue, 10);
+		}
+	}
+	
+	complete_and_exit(&p->thread_exited, 0);
+}
+
+static int prov_init_one(struct async_provider *p)
+{
+	int pid, err;
+
+	init_waitqueue_head(&p->async_wait_queue);
+	
+	p->tfm = crypto_alloc_tfm(async_algo, CRYPTO_TFM_MODE_CBC);
+	if (!p->tfm) {
+		dprintk(KERN_ERR "Failed to allocate %d's %s tfm.\n", p->num, async_algo);
+		return -EINVAL;
+	}
+	
+	err = crypto_cipher_setkey(p->tfm, async_key, sizeof(async_key));
+	if (err) {
+		dprintk("Failed to set key [keylen=%d]: err=%d.\n",
+				sizeof(async_key), err);
+		goto err_out_free_tfm;
+	}
+	
+	init_completion(&p->thread_exited);
+	pid = kernel_thread(async_thread, p, CLONE_FS | CLONE_FILES);
+	if (IS_ERR((void *)pid)) {
+		err = -EINVAL;
+		dprintk(KERN_ERR "Failed to create kernel load balancing thread.\n");
+		goto err_out_free_tfm;
+	}
+	
+	memcpy(&p->pdev, &pdev, sizeof(pdev));
+	snprintf(p->pdev.name, sizeof(p->pdev.name), "async_provider%d", p->num);
+
+	p->pdev.cap_number 	= prov_cap_number;
+	p->pdev.priv 		= p;
+
+	err = crypto_device_add(&p->pdev);
+	if (err)
+		goto err_out_remove_thread;
+
+	return 0;
+
+err_out_remove_thread:
+	need_exit = 1;
+	wake_up(&p->async_wait_queue);
+	wait_for_completion(&p->thread_exited);
+err_out_free_tfm:
+	crypto_free_tfm(p->tfm);
+
+	return err;
+}
+
+static void prov_fini_one(struct async_provider *p)
+{
+	crypto_device_remove(&p->pdev);
+	need_exit = 1;
+	wake_up(&p->async_wait_queue);
+	wait_for_completion(&p->thread_exited);
+
+	crypto_free_tfm(p->tfm);
+}
+
+int prov_init(void)
+{
+	int err, i;
+
+	aprov = kmalloc(trnum * sizeof(struct async_provider), GFP_KERNEL);
+	if (!aprov) {
+		dprintk(KERN_ERR "Failed to allocate %d async_provider pointers.\n", trnum);
+		return -ENOMEM;
+	}
+
+	memset(aprov, 0, trnum * sizeof(struct async_provider));
+
+	for (i=0; i<trnum; ++i) {
+		aprov[i].num = i;
+		
+		err = prov_init_one(&aprov[i]);
+		if (err)
+			goto err_out_fini_one;
+	}
+	
+	dprintk(KERN_INFO "Test crypto provider module %s is loaded for %d processors.\n", 
+			pdev.name, trnum);
+
+	return 0;
+
+err_out_fini_one:
+	i--;
+	while (i >= 0)
+		prov_fini_one(&aprov[i]);
+
+	kfree(aprov);
+
+	return err;
+}
+
+void prov_fini(void)
+{
+	int i;
+	
+	for (i=0; i<trnum; ++i)
+		prov_fini_one(&aprov[i]);
+	
+	kfree(aprov);
+
+	dprintk(KERN_INFO "Test crypto provider module %s is unloaded.\n", pdev.name);
+}
+
+module_init(prov_init);
+module_exit(prov_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Evgeniy Polyakov <[email protected]>");
+MODULE_DESCRIPTION("Test crypto module provider.");

@@ -0,0 +1,160 @@
+/*
+ * 	crypto_conn.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/vmalloc.h>
+#include <linux/connector.h>
+
+#include "acrypto.h"
+#include "crypto_lb.h"
+#include "crypto_conn.h"
+#include "crypto_user_ioctl.h"
+#include "crypto_user_direct.h"
+
+struct cb_id crypto_conn_id = { 0xdead, 0x0000 };
+static char crypto_conn_name[] = "crconn";
+
+static void crypto_conn_callback(void *data)
+{
+	struct cn_msg *msg, *reply;
+	struct crypto_conn_data *d, *cmd;
+	struct crypto_device *dev;
+	u32 sessions;
+
+	msg = (struct cn_msg *)data;
+	d = (struct crypto_conn_data *)msg->data;
+
+	if (msg->len < sizeof(*d)) {
+		dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u < %u.\n",
+			msg->len, sizeof(*d));
+		return;
+	}
+
+	if (msg->len != sizeof(*d) + d->len) {
+		dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u != %u.\n",
+			msg->len, sizeof(*d) + d->len);
+		return;
+	}
+
+	dev = crypto_device_get_name(d->name);
+	if (!dev) {
+		dprintk(KERN_INFO "Crypto device %s was not found.\n", d->name);
+		return;
+	}
+
+	switch (d->cmd) {
+	case CRYPTO_READ_SESSIONS:
+		reply = kmalloc(sizeof(*msg) + sizeof(*cmd) + sizeof(sessions), GFP_ATOMIC);
+		if (reply) {
+			memcpy(reply, msg, sizeof(*reply));
+			reply->len = sizeof(*cmd) + sizeof(sessions);
+
+			/*
+			 * See protocol description in connector.c
+			 */
+			reply->ack++;
+
+			cmd = (struct crypto_conn_data *)(reply + 1);
+			memcpy(cmd, d, sizeof(*cmd));
+			cmd->len = sizeof(sessions);
+
+			sessions = atomic_read(&dev->refcnt);
+
+			memcpy(cmd + 1, &sessions, sizeof(sessions));
+
+			cn_netlink_send(reply, 0);
+
+			kfree(reply);
+		} else
+			dprintk(KERN_ERR "Failed to allocate %d bytes in reply to comamnd 0x%x.\n",
+				sizeof(*msg) + sizeof(*cmd), d->cmd);
+		break;
+	case CRYPTO_GET_STAT:
+		reply = kmalloc(sizeof(*msg) + sizeof(*cmd) + sizeof(struct crypto_device_stat), GFP_ATOMIC);
+		if (reply) {
+			struct crypto_device_stat *ptr;
+
+			memcpy(reply, msg, sizeof(*reply));
+			reply->len = sizeof(*cmd) + sizeof(*ptr);
+
+			/*
+			 * See protocol description in connector.c
+			 */
+			reply->ack++;
+
+			cmd = (struct crypto_conn_data *)(reply + 1);
+			memcpy(cmd, d, sizeof(*cmd));
+			cmd->len = sizeof(*ptr);
+
+			ptr = (struct crypto_device_stat *)(cmd + 1);
+			memcpy(ptr, &dev->stat, sizeof(*ptr));
+
+			cn_netlink_send(reply, 0);
+
+			kfree(reply);
+		} else
+			dprintk(KERN_ERR "Failed to allocate %d bytes in reply to comamnd 0x%x.\n",
+				sizeof(*msg) + sizeof(*cmd), d->cmd);
+		break;
+	case CRYPTO_REQUEST:
+#if 1
+		{
+			struct crypto_user_direct *usr;
+
+			usr = (struct crypto_user_direct *)(d->data);
+			
+			crypto_user_direct_add_request(msg->seq, msg->ack, usr);
+		}
+#endif
+		break;
+	default:
+		dprintk(KERN_ERR "Wrong operation 0x%04x for crypto connector.\n",
+			d->cmd);
+		return;
+	}
+
+	crypto_device_put(dev);
+}
+
+int crypto_conn_init(void)
+{
+	int err;
+
+	err = cn_add_callback(&crypto_conn_id, crypto_conn_name, crypto_conn_callback);
+	if (err)
+		return err;
+
+	dprintk(KERN_INFO "Crypto connector callback is registered.\n");
+
+	return 0;
+}
+
+void crypto_conn_fini(void)
+{
+	cn_del_callback(&crypto_conn_id);
+	dprintk(KERN_INFO "Crypto connector callback is unregistered.\n");
+}

@@ -0,0 +1,45 @@
+/*
+ * 	crypto_conn.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_CONN_H
+#define __CRYPTO_CONN_H
+
+#include "acrypto.h"
+
+#define CRYPTO_READ_SESSIONS			0
+#define CRYPTO_REQUEST				1
+#define CRYPTO_GET_STAT				2
+
+struct crypto_conn_data 
+{
+	char 		name[SCACHE_NAMELEN];
+	__u16 		cmd;
+	__u16 		len;
+	__u8 		data[0];
+};
+
+#ifdef __KERNEL__
+
+int crypto_conn_init(void);
+void crypto_conn_fini(void);
+
+#endif				/* __KERNEL__ */
+#endif				/* __CRYPTO_CONN_H */

@@ -0,0 +1,39 @@
+/*
+ * 	crypto_def.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_DEF_H
+#define __CRYPTO_DEF_H
+
+#define CRYPTO_OP_DECRYPT	0
+#define CRYPTO_OP_ENCRYPT	1
+#define CRYPTO_OP_HMAC		2
+
+#define CRYPTO_MODE_ECB		0
+#define CRYPTO_MODE_CBC		1
+#define CRYPTO_MODE_CFB		2
+#define CRYPTO_MODE_OFB		3
+
+#define CRYPTO_TYPE_AES_128	0
+#define CRYPTO_TYPE_AES_192	1
+#define CRYPTO_TYPE_AES_256	2
+#define CRYPTO_TYPE_3DES	3
+
+#endif				/* __CRYPTO_DEF_H */

@@ -0,0 +1,421 @@
+/*
+ * 	crypto_dev.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/device.h>
+
+#include "acrypto.h"
+
+static LIST_HEAD(cdev_list);
+static spinlock_t cdev_lock = SPIN_LOCK_UNLOCKED;
+static u32 cdev_ids;
+
+struct list_head *crypto_device_list = &cdev_list;
+spinlock_t *crypto_device_lock = &cdev_lock;
+
+static int crypto_match(struct device *dev, struct device_driver *drv)
+{
+	return 1;
+}
+
+static int crypto_probe(struct device *dev)
+{
+	return -ENODEV;
+}
+
+static int crypto_remove(struct device *dev)
+{
+	return 0;
+}
+
+static void crypto_release(struct device *dev)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, device);
+
+	complete(&d->dev_released);
+}
+
+static void crypto_class_release(struct class *class)
+{
+}
+
+static void crypto_class_release_device(struct class_device *class_dev)
+{
+}
+
+struct class crypto_class = {
+	.name 		= "acrypto",
+	.class_release 	= crypto_class_release,
+	.release 	= crypto_class_release_device
+};
+
+struct bus_type crypto_bus_type = {
+	.name 		= "acrypto",
+	.match 		= crypto_match
+};
+
+struct device_driver crypto_driver = {
+	.name 		= "crypto_driver",
+	.bus 		= &crypto_bus_type,
+	.probe 		= crypto_probe,
+	.remove 	= crypto_remove,
+};
+
+struct device crypto_dev = {
+	.parent 	= NULL,
+	.bus 		= &crypto_bus_type,
+	.bus_id		= "Asynchronous crypto",
+	.driver 	= &crypto_driver,
+	.release 	= &crypto_release
+};
+
+static ssize_t sessions_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%d\n", atomic_read(&d->refcnt));
+}
+static ssize_t name_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%s\n", d->name);
+}
+static ssize_t devices_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d;
+	int off = 0;
+
+	spin_lock_irq(&cdev_lock);
+	list_for_each_entry(d, &cdev_list, cdev_entry) {
+		off += sprintf(buf + off, "%s ", d->name);
+	}
+	spin_unlock_irq(&cdev_lock);
+
+	if (!off)
+		off = sprintf(buf, "No devices registered yet.");
+
+	off += sprintf(buf + off, "\n");
+
+	return off;
+}
+
+static ssize_t kmem_failed_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d =  container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%llu\n", d->stat.kmem_failed);
+}
+static ssize_t sstarted_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%llu\n", d->stat.sstarted);
+}
+static ssize_t sfinished_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%llu\n", d->stat.sfinished);
+}
+static ssize_t scompleted_show(struct class_device *dev, char *buf)
+{
+	struct crypto_device *d = container_of(dev, struct crypto_device, class_device);
+
+	return sprintf(buf, "%llu\n", d->stat.scompleted);
+}
+
+static CLASS_DEVICE_ATTR(sessions, 0444, sessions_show, NULL);
+static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL);
+CLASS_DEVICE_ATTR(devices, 0444, devices_show, NULL);
+static CLASS_DEVICE_ATTR(scompleted, 0444, scompleted_show, NULL);
+static CLASS_DEVICE_ATTR(sstarted, 0444, sstarted_show, NULL);
+static CLASS_DEVICE_ATTR(sfinished, 0444, sfinished_show, NULL);
+static CLASS_DEVICE_ATTR(kmem_failed, 0444, kmem_failed_show, NULL);
+
+static int compare_device(struct crypto_device *d1, struct crypto_device *d2)
+{
+	if (!strncmp(d1->name, d2->name, sizeof(d1->name)))
+		return 1;
+
+	return 0;
+}
+
+static void create_device_attributes(struct crypto_device *dev)
+{
+	class_device_create_file(&dev->class_device, &class_device_attr_sessions);
+	class_device_create_file(&dev->class_device, &class_device_attr_name);
+	class_device_create_file(&dev->class_device, &class_device_attr_scompleted);
+	class_device_create_file(&dev->class_device, &class_device_attr_sstarted);
+	class_device_create_file(&dev->class_device, &class_device_attr_sfinished);
+	class_device_create_file(&dev->class_device, &class_device_attr_kmem_failed);
+}
+
+static void remove_device_attributes(struct crypto_device *dev)
+{
+	class_device_remove_file(&dev->class_device, &class_device_attr_sessions);
+	class_device_remove_file(&dev->class_device, &class_device_attr_name);
+	class_device_remove_file(&dev->class_device, &class_device_attr_scompleted);
+	class_device_remove_file(&dev->class_device, &class_device_attr_sstarted);
+	class_device_remove_file(&dev->class_device, &class_device_attr_sfinished);
+	class_device_remove_file(&dev->class_device, &class_device_attr_kmem_failed);
+}
+
+int __match_initializer(struct crypto_capability *cap, struct crypto_session_initializer *ci)
+{
+	dprintk("Match: %04x.%04x.%04x vs. %04x.%04x.%04x.\n",
+			cap->operation, cap->type, cap->mode,
+			ci->operation, ci->type, ci->mode);
+	if (cap->operation == ci->operation && cap->type == ci->type && 
+			cap->mode == (ci->mode & 0x1fff))
+		return 1;
+
+	return 0;
+}
+
+int match_initializer(struct crypto_device *dev, struct crypto_session_initializer *ci)
+{
+	int i;
+
+	for (i = 0; i < dev->cap_number; ++i) {
+		struct crypto_capability *cap = &dev->cap[i];
+
+		if (__match_initializer(cap, ci)) {
+			if (cap->qlen >= atomic_read(&dev->refcnt) + 1) {
+				dprintk("cap->len=%u, req=%u.\n",
+					cap->qlen, atomic_read(&dev->refcnt) + 1);
+				return 1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+void crypto_device_get(struct crypto_device *dev)
+{
+	atomic_inc(&dev->refcnt);
+}
+
+struct crypto_device *crypto_device_get_name(char *name)
+{
+	struct crypto_device *dev;
+	int found = 0;
+
+	spin_lock_irq(&cdev_lock);
+	list_for_each_entry(dev, &cdev_list, cdev_entry) {
+		if (!strcmp(dev->name, name)) {
+			found = 1;
+			crypto_device_get(dev);
+			break;
+		}
+	}
+	spin_unlock_irq(&cdev_lock);
+
+	if (!found)
+		return NULL;
+
+	return dev;
+}
+
+void crypto_device_put(struct crypto_device *dev)
+{
+	atomic_dec(&dev->refcnt);
+}
+
+static int avg_cap_qlen(struct crypto_device *dev)
+{
+	int i, max = 0;
+
+	if (!dev->cap_number)
+		return 0;
+
+	for (i=0; i<dev->cap_number; ++i) {
+		max += dev->cap[i].qlen;
+	}
+
+	return (max / dev->cap_number);
+}
+
+int __crypto_device_add(struct crypto_device *dev)
+{
+	int err, avg;
+
+	memset(&dev->stat, 0, sizeof(dev->stat));
+	spin_lock_init(&dev->stat_lock);
+	spin_lock_init(&dev->lock);
+	spin_lock_init(&dev->session_lock);
+	INIT_LIST_HEAD(&dev->session_list);
+	atomic_set(&dev->refcnt, 0);
+	dev->sid = 0;
+	dev->flags = 0;
+	init_completion(&dev->dev_released);
+	memcpy(&dev->device, &crypto_dev, sizeof(struct device));
+	dev->driver = &crypto_driver;
+
+	dev->session_cache = kmem_cache_create(dev->name, sizeof(struct crypto_session), 
+			0, 0, NULL, NULL);
+	if (!dev->session_cache) {
+		dprintk(KERN_ERR "Failed to create session cache for device %s.\n", dev->name);
+		return -ENOMEM;
+	}
+	
+	avg = avg_cap_qlen(dev);
+
+	if (avg) {
+		dev->session_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->session_cache);
+		if (!dev->session_pool) {
+			dprintk(KERN_ERR "Failed to create memory pool with %d objects for device %s.\n",
+					avg, dev->name);
+			err = -ENOMEM;
+			goto err_out_cache_destroy;
+		}
+	} else
+		dev->session_pool = NULL;
+
+	snprintf(dev->device.bus_id, sizeof(dev->device.bus_id), "%s", dev->name);
+	err = device_register(&dev->device);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto device %s: err=%d.\n",
+			dev->name, err);
+		goto err_out_mempool_destroy;
+	}
+
+	snprintf(dev->class_device.class_id, sizeof(dev->class_device.class_id), "%s", dev->name);
+	dev->class_device.dev = &dev->device;
+	dev->class_device.class = &crypto_class;
+
+	err = class_device_register(&dev->class_device);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto class device %s: err=%d.\n",
+			dev->name, err);
+		goto err_out_device_unregister;
+	}
+
+	create_device_attributes(dev);
+
+	return 0;
+
+err_out_device_unregister:
+	device_unregister(&dev->device);
+err_out_mempool_destroy:
+	if (dev->session_pool)
+		mempool_destroy(dev->session_pool);
+err_out_cache_destroy:
+	kmem_cache_destroy(dev->session_cache);
+
+	return err;
+}
+
+void __crypto_device_remove(struct crypto_device *dev)
+{
+	remove_device_attributes(dev);
+	class_device_unregister(&dev->class_device);
+	device_unregister(&dev->device);
+	if (dev->session_pool)
+		mempool_destroy(dev->session_pool);
+	kmem_cache_destroy(dev->session_cache);
+}
+
+int crypto_device_add(struct crypto_device *dev)
+{
+	int err;
+
+	err = __crypto_device_add(dev);
+	if (err)
+		return err;
+
+	spin_lock_irq(&cdev_lock);
+	list_add(&dev->cdev_entry, &cdev_list);
+	dev->id = ++cdev_ids;
+	spin_unlock_irq(&cdev_lock);
+
+	printk(KERN_INFO "Crypto device %s was registered with ID=%x.\n",
+		dev->name, dev->id);
+
+	return 0;
+}
+
+void crypto_device_remove(struct crypto_device *dev)
+{
+	struct crypto_device *__dev, *n;
+
+	__crypto_device_remove(dev);
+
+	spin_lock_irq(&cdev_lock);
+	list_for_each_entry_safe(__dev, n, &cdev_list, cdev_entry) {
+		if (compare_device(__dev, dev)) {
+			list_del_init(&__dev->cdev_entry);
+			spin_unlock_irq(&cdev_lock);
+
+			/*
+			 * In test cases or when crypto device driver is not written correctly,
+			 * it's ->data_ready() method will not be callen anymore
+			 * after device is removed from crypto device list.
+			 *
+			 * For such cases we either should provide ->flush() call
+			 * or properly write ->data_ready() method.
+			 */
+
+			while (atomic_read(&__dev->refcnt)) {
+
+				dprintk(KERN_INFO "Waiting for %s to become free: refcnt=%d.\n",
+					__dev->name, atomic_read(&dev->refcnt));
+
+				/*
+				 * Hack zone: you need to write good ->data_ready()
+				 * and crypto device driver itself.
+				 *
+				 * Driver shoud not buzz if it has pending sessions
+				 * in it's queue and it was removed from global device list.
+				 *
+				 * Although I can workaround it here, for example by
+				 * flushing the whole queue and drop all pending sessions.
+				 */
+
+				__dev->data_ready(__dev);
+				set_current_state(TASK_UNINTERRUPTIBLE);
+				schedule_timeout(HZ);
+			}
+
+			dprintk(KERN_ERR "Crypto device %s was unregistered.\n",
+				dev->name);
+			return;
+		}
+	}
+	spin_unlock_irq(&cdev_lock);
+
+	dprintk(KERN_ERR "Crypto device %s was not registered.\n", dev->name);
+}
+
+EXPORT_SYMBOL_GPL(crypto_device_add);
+EXPORT_SYMBOL_GPL(crypto_device_remove);
+EXPORT_SYMBOL_GPL(crypto_device_get);
+EXPORT_SYMBOL_GPL(crypto_device_get_name);
+EXPORT_SYMBOL_GPL(crypto_device_put);
+EXPORT_SYMBOL_GPL(match_initializer);

@@ -0,0 +1,634 @@
+/*
+ * 	crypto_lb.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/err.h>
+
+#include "acrypto.h"
+#include "crypto_lb.h"
+#include "crypto_stat.h"
+#include "crypto_route.h"
+
+static LIST_HEAD(crypto_lb_list);
+static spinlock_t crypto_lb_lock = SPIN_LOCK_UNLOCKED;
+static int lb_num = 0;
+static struct crypto_lb *current_lb, *default_lb;
+static struct completion thread_exited;
+static int need_exit;
+static struct workqueue_struct *crypto_lb_queue;
+static DECLARE_WAIT_QUEUE_HEAD(crypto_lb_wait_queue);
+
+extern struct list_head *crypto_device_list;
+extern spinlock_t *crypto_device_lock;
+
+extern int force_lb_remove;
+extern struct crypto_device main_crypto_device;
+
+static int lb_is_current(struct crypto_lb *l)
+{
+	return (l->crypto_device_list != NULL && l->crypto_device_lock != NULL);
+}
+
+static int lb_is_default(struct crypto_lb *l)
+{
+	return (l == default_lb);
+}
+
+static void __lb_set_current(struct crypto_lb *l)
+{
+	struct crypto_lb *c = current_lb;
+
+	if (c) {
+		l->crypto_device_list = crypto_device_list;
+		l->crypto_device_lock = crypto_device_lock;
+		current_lb = l;
+		c->crypto_device_list = NULL;
+		c->crypto_device_lock = NULL;
+	} else {
+		l->crypto_device_list = crypto_device_list;
+		l->crypto_device_lock = crypto_device_lock;
+		current_lb = l;
+	}
+}
+
+static void lb_set_current(struct crypto_lb *l)
+{
+	struct crypto_lb *c = current_lb;
+
+	if (c) {
+		spin_lock_irq(&c->lock);
+		__lb_set_current(l);
+		spin_unlock_irq(&c->lock);
+	} else
+		__lb_set_current(l);
+}
+
+static void __lb_set_default(struct crypto_lb *l)
+{
+	default_lb = l;
+}
+
+static void lb_set_default(struct crypto_lb *l)
+{
+	struct crypto_lb *c = default_lb;
+
+	if (c) {
+		spin_lock_irq(&c->lock);
+		__lb_set_default(l);
+		spin_unlock_irq(&c->lock);
+	} else
+		__lb_set_default(l);
+}
+
+static int crypto_lb_match(struct device *dev, struct device_driver *drv)
+{
+	return 1;
+}
+
+static int crypto_lb_probe(struct device *dev)
+{
+	return -ENODEV;
+}
+
+static int crypto_lb_remove(struct device *dev)
+{
+	return 0;
+}
+
+static void crypto_lb_release(struct device *dev)
+{
+	struct crypto_lb *d = container_of(dev, struct crypto_lb, device);
+
+	complete(&d->dev_released);
+}
+
+static void crypto_lb_class_release(struct class *class)
+{
+}
+
+static void crypto_lb_class_release_device(struct class_device *class_dev)
+{
+}
+
+struct class crypto_lb_class = {
+	.name 		= "crypto_lb",
+	.class_release 	= crypto_lb_class_release,
+	.release 	= crypto_lb_class_release_device
+};
+
+struct bus_type crypto_lb_bus_type = {
+	.name 		= "crypto_lb",
+	.match 		= crypto_lb_match
+};
+
+struct device_driver crypto_lb_driver = {
+	.name 		= "crypto_lb_driver",
+	.bus 		= &crypto_lb_bus_type,
+	.probe 		= crypto_lb_probe,
+	.remove 	= crypto_lb_remove,
+};
+
+struct device crypto_lb_dev = {
+	.parent 	= NULL,
+	.bus 		= &crypto_lb_bus_type,
+	.bus_id 	= "crypto load balancer",
+	.driver 	= &crypto_lb_driver,
+	.release 	= &crypto_lb_release
+};
+
+static ssize_t name_show(struct class_device *dev, char *buf)
+{
+	struct crypto_lb *lb = container_of(dev, struct crypto_lb, class_device);
+
+	return sprintf(buf, "%s\n", lb->name);
+}
+
+static ssize_t current_show(struct class_device *dev, char *buf)
+{
+	struct crypto_lb *lb;
+	int off = 0;
+
+	spin_lock_irq(&crypto_lb_lock);
+
+	list_for_each_entry(lb, &crypto_lb_list, lb_entry) {
+		if (lb_is_current(lb))
+			off += sprintf(buf + off, "[");
+		if (lb_is_default(lb))
+			off += sprintf(buf + off, "(");
+		off += sprintf(buf + off, "%s", lb->name);
+		if (lb_is_default(lb))
+			off += sprintf(buf + off, ")");
+		if (lb_is_current(lb))
+			off += sprintf(buf + off, "]");
+	}
+
+	spin_unlock_irq(&crypto_lb_lock);
+
+	if (!off)
+		off = sprintf(buf, "No load balancers regitered yet.");
+
+	off += sprintf(buf + off, "\n");
+
+	return off;
+}
+static ssize_t current_store(struct class_device *dev, const char *buf, size_t count)
+{
+	struct crypto_lb *lb;
+
+	spin_lock_irq(&crypto_lb_lock);
+
+	list_for_each_entry(lb, &crypto_lb_list, lb_entry) {
+		if (count == strlen(lb->name) && !strcmp(buf, lb->name)) {
+			lb_set_current(lb);
+			lb_set_default(lb);
+
+			dprintk(KERN_INFO "Load balancer %s is set as current and default.\n",
+				lb->name);
+
+			break;
+		}
+	}
+	spin_unlock_irq(&crypto_lb_lock);
+
+	return count;
+}
+
+static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL);
+CLASS_DEVICE_ATTR(lbs, 0644, current_show, current_store);
+
+static void create_device_attributes(struct crypto_lb *lb)
+{
+	class_device_create_file(&lb->class_device, &class_device_attr_name);
+}
+
+static void remove_device_attributes(struct crypto_lb *lb)
+{
+	class_device_remove_file(&lb->class_device, &class_device_attr_name);
+}
+
+static int compare_lb(struct crypto_lb *l1, struct crypto_lb *l2)
+{
+	if (!strncmp(l1->name, l2->name, sizeof(l1->name)))
+		return 1;
+
+	return 0;
+}
+
+void crypto_lb_rehash(void)
+{
+	if (!current_lb)
+		return;
+
+	spin_lock_irq(&current_lb->lock);
+
+	current_lb->rehash(current_lb);
+
+	spin_unlock_irq(&current_lb->lock);
+
+	wake_up_interruptible(&crypto_lb_wait_queue);
+}
+
+struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *ci, struct crypto_data *data)
+{
+	struct crypto_device *dev;
+
+	if (!current_lb)
+		return NULL;
+
+	if (sci_binded(ci)) {
+		int found = 0;
+
+		spin_lock_irq(crypto_device_lock);
+
+		list_for_each_entry(dev, crypto_device_list, cdev_entry) {
+			if (dev->id == ci->bdev) {
+				found = 1;
+				break;
+			}
+		}
+
+		spin_unlock_irq(crypto_device_lock);
+
+		return (found) ? dev : NULL;
+	}
+
+	spin_lock_irq(&current_lb->lock);
+
+	current_lb->rehash(current_lb);
+
+	spin_lock(crypto_device_lock);
+
+	dev = current_lb->find_device(current_lb, ci, data);
+	if (dev)
+		crypto_device_get(dev);
+
+	spin_unlock(crypto_device_lock);
+
+	spin_unlock_irq(&current_lb->lock);
+
+	wake_up_interruptible(&crypto_lb_wait_queue);
+
+	return dev;
+}
+
+static int __crypto_lb_register(struct crypto_lb *lb)
+{
+	int err;
+
+	spin_lock_init(&lb->lock);
+
+	init_completion(&lb->dev_released);
+	memcpy(&lb->device, &crypto_lb_dev, sizeof(struct device));
+	lb->driver = &crypto_lb_driver;
+
+	snprintf(lb->device.bus_id, sizeof(lb->device.bus_id), "%s", lb->name);
+	err = device_register(&lb->device);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto load balancer device %s: err=%d.\n",
+			lb->name, err);
+		return err;
+	}
+
+	snprintf(lb->class_device.class_id, sizeof(lb->class_device.class_id), "%s", lb->name);
+	lb->class_device.dev = &lb->device;
+	lb->class_device.class = &crypto_lb_class;
+
+	err = class_device_register(&lb->class_device);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto load balancer class device %s: err=%d.\n",
+			lb->name, err);
+		device_unregister(&lb->device);
+		return err;
+	}
+
+	create_device_attributes(lb);
+	wake_up_interruptible(&crypto_lb_wait_queue);
+
+	return 0;
+
+}
+
+static void __crypto_lb_unregister(struct crypto_lb *lb)
+{
+	wake_up_interruptible(&crypto_lb_wait_queue);
+	remove_device_attributes(lb);
+	class_device_unregister(&lb->class_device);
+	device_unregister(&lb->device);
+}
+
+int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default)
+{
+	struct crypto_lb *__lb;
+	int err;
+
+	spin_lock_irq(&crypto_lb_lock);
+
+	list_for_each_entry(__lb, &crypto_lb_list, lb_entry) {
+		if (unlikely(compare_lb(__lb, lb))) {
+			spin_unlock_irq(&crypto_lb_lock);
+
+			dprintk(KERN_ERR "Crypto load balancer %s is already registered.\n",
+				lb->name);
+			return -EINVAL;
+		}
+	}
+
+	list_add(&lb->lb_entry, &crypto_lb_list);
+
+	spin_unlock_irq(&crypto_lb_lock);
+
+	err = __crypto_lb_register(lb);
+	if (err) {
+		spin_lock_irq(&crypto_lb_lock);
+		list_del_init(&lb->lb_entry);
+		spin_unlock_irq(&crypto_lb_lock);
+
+		return err;
+	}
+
+	if (!default_lb || set_default)
+		lb_set_default(lb);
+
+	if (!current_lb || set_current)
+		lb_set_current(lb);
+
+	dprintk(KERN_INFO "Crypto load balancer %s was registered and set to be [%s.%s].\n",
+		lb->name, (lb_is_current(lb)) ? "current" : "not current",
+		(lb_is_default(lb)) ? "default" : "not default");
+
+	lb_num++;
+
+	return 0;
+}
+
+void crypto_lb_unregister(struct crypto_lb *lb)
+{
+	struct crypto_lb *__lb, *n;
+
+	if (lb_num == 1) {
+		dprintk(KERN_INFO "You are removing crypto load balancer %s which is current and default.\n"
+			"There is no other crypto load balancers. "
+			"Removing %s delayed untill new load balancer is registered.\n",
+			lb->name, (force_lb_remove) ? "is not" : "is");
+		while (lb_num == 1 && !force_lb_remove) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(HZ);
+
+			if (signal_pending(current))
+				flush_signals(current);
+		}
+	}
+
+	__crypto_lb_unregister(lb);
+
+	spin_lock_irq(&crypto_lb_lock);
+
+	list_for_each_entry_safe(__lb, n, &crypto_lb_list, lb_entry) {
+		if (compare_lb(__lb, lb)) {
+			lb_num--;
+			list_del_init(&__lb->lb_entry);
+
+			dprintk(KERN_ERR "Crypto load balancer %s was unregistered.\n",
+				lb->name);
+		} else if (lb_num) {
+			if (lb_is_default(lb))
+				lb_set_default(__lb);
+			if (lb_is_current(lb))
+				lb_set_current(default_lb);
+		}
+	}
+
+	spin_unlock_irq(&crypto_lb_lock);
+}
+
+static void crypto_lb_queue_wrapper(void *data)
+{
+	struct crypto_device *dev = &main_crypto_device;
+	struct crypto_session *s = (struct crypto_session *)data;
+
+	dprintk(KERN_INFO "%s: Calling callback for session %llu [%llu] flags=%x, "
+		"op=%04u, type=%04x, mode=%04x, priority=%04x\n", __func__,
+		s->ci.id, s->ci.dev_id, s->ci.flags, s->ci.operation,
+		s->ci.type, s->ci.mode, s->ci.priority);
+
+	spin_lock_irq(&s->lock);
+	crypto_stat_finish_inc(s);
+
+	finish_session(s);
+	unstart_session(s);
+	spin_unlock_irq(&s->lock);
+
+	s->ci.callback(&s->ci, &s->data);
+
+	if (session_finished(s)) {
+		crypto_session_destroy(s);
+		return;
+	} else {
+		/*
+		 * Special case: crypto consumer marks session as "not finished"
+		 * in it's callback - it means that crypto consumer wants 
+		 * this session to be processed further, 
+		 * for example crypto consumer can add new route and then
+		 * mark session as "not finished".
+		 */
+
+		uncomplete_session(s);
+		unstart_session(s);
+		crypto_session_insert_main(dev, s);
+	}
+	spin_unlock_irq(&s->lock);
+}
+
+static void crypto_lb_process_next_route(struct crypto_session *s)
+{
+	struct crypto_route *rt;
+	struct crypto_device *dev, *orig;
+
+	rt = crypto_route_dequeue(s);
+	if (rt) {
+		orig = rt->dev;
+
+		list_del_init(&s->dev_queue_entry);
+
+		crypto_route_free(rt);
+
+		dev = crypto_route_get_current_device(s);
+		if (dev) {
+			dprintk(KERN_INFO "%s: processing new route to %s.\n",
+				__func__, dev->name);
+
+			memcpy(&s->ci, &rt->ci, sizeof(s->ci));
+
+			if (!strncmp(orig->name, dev->name, sizeof(dev->name)))
+				__crypto_session_insert(dev, s);
+			else
+				crypto_session_insert(dev, s);
+
+			/*
+			 * Reference to this device was already hold when
+			 * new routing was added.
+			 */
+			crypto_device_put(dev);
+		}
+	}
+}
+
+void crypto_wake_lb(void)
+{
+	wake_up_interruptible(&crypto_lb_wait_queue);
+}
+
+int crypto_lb_thread(void *data)
+{
+	struct crypto_session *s, *n;
+	struct crypto_device *dev = (struct crypto_device *)data;
+	unsigned long flags;
+
+	daemonize("%s", dev->name);
+	allow_signal(SIGTERM);
+
+	while (!need_exit) {
+		spin_lock_irqsave(&dev->session_lock, flags);
+		list_for_each_entry_safe(s, n, &dev->session_list, main_queue_entry) {
+			dprintk("session %llu [%llu]: flags=%x, route_num=%d, %s,%s,%s,%s.\n",
+			     s->ci.id, s->ci.dev_id, s->ci.flags,
+			     crypto_route_queue_len(s),
+			     (session_completed(s)) ? "completed" : "not completed",
+			     (session_finished(s)) ? "finished" : "not finished",
+			     (session_started(s)) ? "started" : "not started",
+			     (session_is_processed(s)) ? "is being processed" : "is not being processed");
+
+			if (!spin_trylock(&s->lock))
+				continue;
+
+			if (session_is_processed(s))
+				goto unlock;
+			if (session_started(s))
+				goto unlock;
+
+			if (session_completed(s)) {
+				crypto_stat_ptime_inc(s);
+
+				if (crypto_route_queue_len(s) > 1) {
+					crypto_lb_process_next_route(s);
+				} else {
+					start_session(s);
+					crypto_stat_start_inc(s);
+
+					dprintk("%s: going to remove session %llu [%llu].\n",
+					     __func__, s->ci.id, s->ci.dev_id);
+
+					__crypto_session_dequeue_main(s);
+					spin_unlock(&s->lock);
+
+					INIT_WORK(&s->work, &crypto_lb_queue_wrapper, s);
+					queue_work(crypto_lb_queue, &s->work);
+					continue;
+				}
+			}
+unlock:
+			spin_unlock(&s->lock);
+		}
+		spin_unlock_irqrestore(&dev->session_lock, flags);
+
+		interruptible_sleep_on_timeout(&crypto_lb_wait_queue, 100);
+	}
+
+	flush_workqueue(crypto_lb_queue);
+	complete_and_exit(&thread_exited, 0);
+}
+
+int crypto_lb_init(void)
+{
+	int err;
+	long pid;
+
+	err = bus_register(&crypto_lb_bus_type);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto load balancer bus: err=%d.\n", err);
+		goto err_out_exit;
+	}
+
+	err = driver_register(&crypto_lb_driver);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto load balancer driver: err=%d.\n", err);
+		goto err_out_bus_unregister;
+	}
+
+	crypto_lb_class.class_dev_attrs = &class_device_attr_lbs;
+
+	err = class_register(&crypto_lb_class);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto load balancer class: err=%d.\n", err);
+		goto err_out_driver_unregister;
+	}
+
+	crypto_lb_queue = create_workqueue("clbq");
+	if (!crypto_lb_queue) {
+		dprintk(KERN_ERR "Failed to create crypto load balaner work queue.\n");
+		goto err_out_class_unregister;
+	}
+
+	init_completion(&thread_exited);
+	pid = kernel_thread(crypto_lb_thread, &main_crypto_device, CLONE_FS | CLONE_FILES);
+	if (IS_ERR((void *)pid)) {
+		dprintk(KERN_ERR "Failed to create kernel load balancing thread.\n");
+		goto err_out_destroy_workqueue;
+	}
+
+	return 0;
+
+err_out_destroy_workqueue:
+	destroy_workqueue(crypto_lb_queue);
+err_out_class_unregister:
+	class_unregister(&crypto_lb_class);
+err_out_driver_unregister:
+	driver_unregister(&crypto_lb_driver);
+err_out_bus_unregister:
+	bus_unregister(&crypto_lb_bus_type);
+err_out_exit:
+	return err;
+}
+
+void crypto_lb_fini(void)
+{
+	need_exit = 1;
+	wait_for_completion(&thread_exited);
+	flush_workqueue(crypto_lb_queue);
+	destroy_workqueue(crypto_lb_queue);
+	class_unregister(&crypto_lb_class);
+	driver_unregister(&crypto_lb_driver);
+	bus_unregister(&crypto_lb_bus_type);
+}
+
+EXPORT_SYMBOL_GPL(crypto_lb_register);
+EXPORT_SYMBOL_GPL(crypto_lb_unregister);
+EXPORT_SYMBOL_GPL(crypto_lb_rehash);
+EXPORT_SYMBOL_GPL(crypto_lb_find_device);
+EXPORT_SYMBOL_GPL(crypto_wake_lb);

@@ -0,0 +1,63 @@
+/*
+ * 	crypto_lb.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_LB_H
+#define __CRYPTO_LB_H
+
+#include "acrypto.h"
+
+#define CRYPTO_LB_NAMELEN	32
+
+struct crypto_lb 
+{
+	struct list_head 	lb_entry;
+
+	char 			name[CRYPTO_LB_NAMELEN];
+
+	void 			(*rehash)(struct crypto_lb *);
+	struct crypto_device *	(*find_device) (struct crypto_lb *,
+						struct crypto_session_initializer *, 
+						struct crypto_data *);
+
+	spinlock_t 		lock;
+
+	spinlock_t 		*crypto_device_lock;
+	struct list_head 	*crypto_device_list;
+
+	struct device_driver 	*driver;
+	struct device 		device;
+	struct class_device 	class_device;
+	struct completion 	dev_released;
+
+};
+
+int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default);
+void crypto_lb_unregister(struct crypto_lb *);
+
+inline void crypto_lb_rehash(void);
+struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *, struct crypto_data *);
+
+void crypto_wake_lb(void);
+
+int crypto_lb_init(void);
+void crypto_lb_fini(void);
+
+#endif				/* __CRYPTO_LB_H */

@@ -0,0 +1,374 @@
+/*
+ * 	crypto_main.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+
+#include "acrypto.h"
+#include "crypto_lb.h"
+#include "crypto_conn.h"
+#include "crypto_route.h"
+#include "crypto_user_ioctl.h"
+
+int force_lb_remove;
+module_param(force_lb_remove, int, 0);
+
+struct crypto_device main_crypto_device;
+
+extern struct bus_type crypto_bus_type;
+extern struct device_driver crypto_driver;
+extern struct class crypto_class;
+extern struct device crypto_dev;
+
+extern struct class_device_attribute class_device_attr_devices;
+extern struct class_device_attribute class_device_attr_lbs;
+
+static void dump_ci(struct crypto_session_initializer *ci)
+{
+	dprintk("%llu [%llu] op=%04u, type=%04x, mode=%04x, priority=%04x",
+		ci->id, ci->dev_id,
+		ci->operation, ci->type, ci->mode, ci->priority);
+}
+
+void __crypto_session_insert(struct crypto_device *dev, struct crypto_session *s)
+{
+	struct crypto_session *__s;
+
+	if (unlikely(list_empty(&dev->session_list))) {
+		list_add(&s->dev_queue_entry, &dev->session_list);
+	} else {
+		int inserted = 0;
+
+		list_for_each_entry(__s, &dev->session_list, dev_queue_entry) {
+			if (__s->ci.priority < s->ci.priority) {
+				list_add_tail(&s->dev_queue_entry, &__s->dev_queue_entry);
+				inserted = 1;
+				break;
+			}
+		}
+
+		if (!inserted)
+			list_add_tail(&s->dev_queue_entry, &dev->session_list);
+	}
+
+	dump_ci(&s->ci);
+	dprintk(" added to crypto device %s [%d].\n", dev->name, atomic_read(&dev->refcnt));
+}
+
+void crypto_session_insert_main(struct crypto_device *dev, struct crypto_session *s)
+{
+	struct crypto_session *__s;
+
+	spin_lock_irq(&dev->session_lock);
+
+	crypto_device_get(dev);
+	if (unlikely(list_empty(&dev->session_list))) {
+		list_add(&s->main_queue_entry, &dev->session_list);
+	} else {
+		int inserted = 0;
+
+		list_for_each_entry(__s, &dev->session_list, main_queue_entry) {
+			if (__s->ci.priority < s->ci.priority) {
+				list_add_tail(&s->main_queue_entry,
+					      &__s->main_queue_entry);
+				inserted = 1;
+				break;
+			}
+		}
+
+		if (!inserted)
+			list_add_tail(&s->main_queue_entry, &dev->session_list);
+	}
+
+	spin_unlock_irq(&dev->session_lock);
+}
+
+void crypto_session_insert(struct crypto_device *dev, struct crypto_session *s)
+{
+	spin_lock_irq(&dev->session_lock);
+	__crypto_session_insert(dev, s);
+	spin_unlock_irq(&dev->session_lock);
+}
+
+void crypto_session_destroy(struct crypto_session *s)
+{
+	if (s->data.priv_size && s->data.priv)
+		kfree(s->data.priv);
+
+	if (session_from_cache(s))
+		kmem_cache_free(s->pool_dev->session_cache, s);
+	else
+		mempool_free(s, s->pool_dev->session_pool);
+}
+
+struct crypto_session *crypto_session_create(struct crypto_session_initializer *ci, struct crypto_data *d)
+{
+	struct crypto_device *dev = &main_crypto_device;
+	struct crypto_device *ldev;
+	struct crypto_session *s;
+	int err;
+
+	if (d->priv_size > CRYPTO_MAX_PRIV_SIZE) {
+		dprintk("priv_size %u is too big, maximum allowed %u.\n",
+			d->priv_size, CRYPTO_MAX_PRIV_SIZE);
+		return NULL;
+	}
+	
+	ldev = crypto_lb_find_device(ci, d);
+	if (!ldev) {
+		dprintk("Cannot find suitable device for [%02x.%02x.%02x.%02x].\n",
+				ci->operation, ci->mode, ci->type, ci->priority);
+		return NULL;
+	}
+	
+	s = mempool_alloc(ldev->session_pool, GFP_ATOMIC);
+	if (!s) {
+		ldev->stat.pool_failed++;
+
+		s = kmem_cache_alloc(ldev->session_cache, GFP_ATOMIC);
+		if (!s) {
+			ldev->stat.kmem_failed++;
+			goto err_out_device_put;
+		}
+
+		mark_session_from_cache(s);
+	}
+	
+	s->pool_dev = ldev;
+
+	crypto_route_head_init(&s->route_list);
+	INIT_LIST_HEAD(&s->dev_queue_entry);
+	INIT_LIST_HEAD(&s->main_queue_entry);
+
+	spin_lock_init(&s->lock);
+
+	memcpy(&s->ci, ci, sizeof(s->ci));
+	memcpy(&s->data, d, sizeof(s->data));
+
+	s->data.priv = NULL;
+	if (d->priv_size) {
+		s->data.priv = kmalloc(d->priv_size, GFP_ATOMIC);
+		if (!s->data.priv)
+			goto err_out_session_free;
+
+		if (d->priv)
+			memcpy(s->data.priv, d->priv, d->priv_size);
+	}
+	else
+		s->data.priv = d->priv;
+
+	s->ci.id = dev->sid++;
+	s->ci.dev_id = ldev->sid++;
+	s->ci.flags = 0;
+
+	err = crypto_route_add_direct(ldev, s, ci);
+	if (err) {
+		dprintk("Can not add route to device %s.\n", ldev->name);
+		goto err_out_session_free;
+	}
+
+	return s;
+
+err_out_session_free:
+	crypto_session_destroy(s);
+err_out_device_put:
+	crypto_device_put(ldev);
+
+	return NULL;
+}
+
+void crypto_session_add(struct crypto_session *s)
+{
+	struct crypto_device *ldev;
+	struct crypto_device *dev = &main_crypto_device;
+
+	ldev = crypto_route_get_current_device(s);
+	BUG_ON(!ldev);		/* This can not happen. */
+
+	spin_lock_irq(&s->lock);
+	crypto_session_insert(ldev, s);
+	crypto_device_put(ldev);
+	crypto_session_insert_main(dev, s);
+	spin_unlock_irq(&s->lock);
+
+	if (ldev->data_ready)
+		ldev->data_ready(ldev);
+}
+
+struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *ci, struct crypto_data *d)
+{
+	struct crypto_session *s;
+
+	s = crypto_session_create(ci, d);
+	if (!s)
+		return NULL;
+
+	crypto_session_add(s);
+
+	return s;
+}
+
+void crypto_session_dequeue_route(struct crypto_session *s)
+{
+	struct crypto_route *rt;
+	struct crypto_device *dev;
+
+	BUG_ON(crypto_route_queue_len(s) > 1);
+
+	while ((rt = crypto_route_dequeue(s))) {
+		dev = rt->dev;
+
+		dprintk(KERN_INFO "Removing route entry for device %s.\n", dev->name);
+
+		spin_lock_irq(&dev->session_lock);
+		list_del_init(&s->dev_queue_entry);
+		spin_unlock_irq(&dev->session_lock);
+
+		crypto_route_free(rt);
+	}
+}
+
+void __crypto_session_dequeue_main(struct crypto_session *s)
+{
+	struct crypto_device *dev = &main_crypto_device;
+
+	list_del(&s->main_queue_entry);
+	crypto_device_put(dev);
+}
+
+void crypto_session_dequeue_main(struct crypto_session *s)
+{
+	struct crypto_device *dev = &main_crypto_device;
+
+	spin_lock_irq(&dev->session_lock);
+
+	__crypto_session_dequeue_main(s);
+
+	spin_unlock_irq(&dev->session_lock);
+}
+
+int __devinit cmain_init(void)
+{
+	struct crypto_device *dev = &main_crypto_device;
+	int err;
+
+	snprintf(dev->name, sizeof(dev->name), "crypto_sessions");
+
+	err = bus_register(&crypto_bus_type);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto bus: err=%d.\n",
+			err);
+		return err;
+	}
+
+	err = driver_register(&crypto_driver);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto driver: err=%d.\n",
+			err);
+		goto err_out_bus_unregister;
+	}
+
+	err = class_register(&crypto_class);
+	if (err) {
+		dprintk(KERN_ERR "Failed to register crypto class: err=%d.\n",
+			err);
+		goto err_out_driver_unregister;
+	}
+
+	err = crypto_lb_init();
+	if (err)
+		goto err_out_class_unregister;
+
+	err = crypto_conn_init();
+	if (err)
+		goto err_out_crypto_lb_fini;
+
+	err = __crypto_device_add(dev);
+	if (err)
+		goto err_out_crypto_conn_fini;
+
+	err = class_device_create_file(&dev->class_device, &class_device_attr_devices);
+	if (err)
+		dprintk("Failed to create \"devices\" attribute: err=%d.\n", err);
+
+	err = class_device_create_file(&dev->class_device, &class_device_attr_lbs);
+	if (err)
+		dprintk("Failed to create \"lbs\" attribute: err=%d.\n", err);
+
+	err = crypto_user_ioctl_init();
+	if (err)
+		goto err_out_remove_files;
+
+	return 0;
+
+err_out_remove_files:
+	class_device_remove_file(&dev->class_device, &class_device_attr_devices);
+	class_device_remove_file(&dev->class_device, &class_device_attr_lbs);
+	__crypto_device_remove(dev);
+err_out_crypto_conn_fini:
+	crypto_conn_fini();
+err_out_crypto_lb_fini:
+	crypto_lb_fini();
+err_out_class_unregister:
+	class_unregister(&crypto_class);
+err_out_driver_unregister:
+	driver_unregister(&crypto_driver);
+err_out_bus_unregister:
+	bus_unregister(&crypto_bus_type);
+
+	return err;
+}
+
+void __devexit cmain_fini(void)
+{
+	struct crypto_device *dev = &main_crypto_device;
+
+	crypto_user_ioctl_fini();
+	
+	class_device_remove_file(&dev->class_device, &class_device_attr_devices);
+	class_device_remove_file(&dev->class_device, &class_device_attr_lbs);
+	__crypto_device_remove(dev);
+
+	crypto_conn_fini();
+	crypto_lb_fini();
+
+	class_unregister(&crypto_class);
+	driver_unregister(&crypto_driver);
+	bus_unregister(&crypto_bus_type);
+}
+
+module_init(cmain_init);
+module_exit(cmain_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Evgeniy Polyakov <[email protected]>");
+MODULE_DESCRIPTION("Asynchronous crypto layer.");
+
+EXPORT_SYMBOL(crypto_session_alloc);
+EXPORT_SYMBOL_GPL(crypto_session_create);
+EXPORT_SYMBOL_GPL(crypto_session_add);
+EXPORT_SYMBOL_GPL(crypto_session_dequeue_route);

@@ -0,0 +1,242 @@
+/*
+ * 	crypto_route.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_ROUTE_H
+#define __CRYPTO_ROUTE_H
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "acrypto.h"
+
+static inline struct crypto_route *crypto_route_alloc_direct(struct crypto_device *dev,
+							     struct crypto_session_initializer *ci)
+{
+	struct crypto_route *rt;
+
+	rt = kmalloc(sizeof(*rt), GFP_ATOMIC);
+	if (!rt) {
+		crypto_device_put(dev);
+		return NULL;
+	}
+
+	memset(rt, 0, sizeof(*rt));
+	memcpy(&rt->ci, ci, sizeof(*ci));
+
+	rt->dev = dev;
+
+	return rt;
+}
+
+static inline struct crypto_route *crypto_route_alloc(struct crypto_device *dev,
+							struct crypto_session_initializer *ci)
+{
+	struct crypto_route *rt;
+
+	if (!match_initializer(dev, ci))
+		return NULL;
+
+	rt = crypto_route_alloc_direct(dev, ci);
+
+	return rt;
+}
+
+static inline void crypto_route_free(struct crypto_route *rt)
+{
+	crypto_device_put(rt->dev);
+	rt->dev = NULL;
+	kfree(rt);
+}
+
+static inline void __crypto_route_del(struct crypto_route *rt, struct crypto_route_head *list)
+{
+	struct crypto_route *next, *prev;
+
+	list->qlen--;
+	next = rt->next;
+	prev = rt->prev;
+	rt->next = rt->prev = NULL;
+	rt->list = NULL;
+	next->prev = prev;
+	prev->next = next;
+}
+
+static inline void crypto_route_del(struct crypto_route *rt)
+{
+	struct crypto_route_head *list = rt->list;
+
+	if (list) {
+		spin_lock_irq(&list->lock);
+		if (list == rt->list)
+			__crypto_route_del(rt, rt->list);
+		spin_unlock_irq(&list->lock);
+
+		crypto_route_free(rt);
+	}
+}
+
+static inline struct crypto_route *__crypto_route_dequeue(struct crypto_route_head *list)
+{
+	struct crypto_route *next, *prev, *result;
+
+	prev = (struct crypto_route *)list;
+	next = prev->next;
+	result = NULL;
+	if (next != prev) {
+		result = next;
+		next = next->next;
+		list->qlen--;
+		next->prev = prev;
+		prev->next = next;
+		result->next = result->prev = NULL;
+		result->list = NULL;
+	}
+	return result;
+}
+
+static inline struct crypto_route *crypto_route_dequeue(struct crypto_session *s)
+{
+	struct crypto_route *rt;
+
+	spin_lock_irq(&s->route_list.lock);
+
+	rt = __crypto_route_dequeue(&s->route_list);
+
+	spin_unlock_irq(&s->route_list.lock);
+
+	return rt;
+}
+
+static inline void __crypto_route_queue(struct crypto_route *rt, struct crypto_route_head *list)
+{
+	struct crypto_route *prev, *next;
+
+	rt->list = list;
+	list->qlen++;
+	next = (struct crypto_route *)list;
+	prev = next->prev;
+	rt->next = next;
+	rt->prev = prev;
+	next->prev = prev->next = rt;
+}
+
+static inline void crypto_route_queue(struct crypto_route *rt, struct crypto_session *s)
+{
+
+	spin_lock_irq(&s->route_list.lock);
+
+	__crypto_route_queue(rt, &s->route_list);
+
+	spin_unlock_irq(&s->route_list.lock);
+}
+
+static inline int crypto_route_add(struct crypto_device *dev, struct crypto_session *s, 
+						struct crypto_session_initializer *ci)
+{
+	struct crypto_route *rt;
+
+	rt = crypto_route_alloc(dev, ci);
+	if (!rt)
+		return -ENOMEM;
+
+	crypto_route_queue(rt, s);
+
+	return 0;
+}
+
+static inline int crypto_route_add_direct(struct crypto_device *dev, struct crypto_session *s,
+						struct crypto_session_initializer *ci)
+{
+	struct crypto_route *rt;
+
+	rt = crypto_route_alloc_direct(dev, ci);
+	if (!rt)
+		return -ENOMEM;
+
+	crypto_route_queue(rt, s);
+
+	return 0;
+}
+
+static inline int crypto_route_queue_len(struct crypto_session *s)
+{
+	return s->route_list.qlen;
+}
+
+static inline void crypto_route_head_init(struct crypto_route_head *list)
+{
+	spin_lock_init(&list->lock);
+	list->prev = list->next = (struct crypto_route *)list;
+	list->qlen = 0;
+}
+
+static inline struct crypto_route *__crypto_route_current(struct crypto_route_head *list)
+{
+	struct crypto_route *next, *prev, *result;
+
+	prev = (struct crypto_route *)list;
+	next = prev->next;
+	result = NULL;
+	if (next != prev)
+		result = next;
+
+	return result;
+}
+
+static inline struct crypto_route *crypto_route_current(struct crypto_session *s)
+{
+	struct crypto_route_head *list;
+	struct crypto_route *rt = NULL;
+
+	list = &s->route_list;
+
+	if (list) {
+		spin_lock_irq(&list->lock);
+
+		rt = __crypto_route_current(list);
+
+		spin_unlock_irq(&list->lock);
+	}
+
+	return rt;
+}
+
+static inline struct crypto_device *crypto_route_get_current_device(struct crypto_session *s)
+{
+	struct crypto_route *rt = NULL;
+	struct crypto_device *dev = NULL;
+	struct crypto_route_head *list = &s->route_list;
+
+	spin_lock_irq(&list->lock);
+
+	rt = __crypto_route_current(list);
+	if (rt) {
+		dev = rt->dev;
+		crypto_device_get(dev);
+	}
+
+	spin_unlock_irq(&list->lock);
+
+	return dev;
+}
+
+#endif				/* __CRYPTO_ROUTE_H */

@@ -0,0 +1,100 @@
+/*
+ * 	crypto_stat.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+
+#include "acrypto.h"
+#include "crypto_route.h"
+
+void crypto_stat_start_inc(struct crypto_session *s)
+{
+	struct crypto_device *dev;
+
+	dev = crypto_route_get_current_device(s);
+	if (dev) {
+		spin_lock_irq(&dev->stat_lock);
+		dev->stat.sstarted++;
+		spin_unlock_irq(&dev->stat_lock);
+
+		crypto_device_put(dev);
+	}
+}
+
+void crypto_stat_finish_inc(struct crypto_session *s)
+{
+	struct crypto_device *dev;
+
+	dev = crypto_route_get_current_device(s);
+	if (dev) {
+		spin_lock_irq(&dev->stat_lock);
+		dev->stat.sfinished++;
+		spin_unlock_irq(&dev->stat_lock);
+
+		crypto_device_put(dev);
+	}
+}
+
+void crypto_stat_complete_inc(struct crypto_session *s)
+{
+	struct crypto_device *dev;
+
+	dev = crypto_route_get_current_device(s);
+	if (dev) {
+		spin_lock_irq(&dev->stat_lock);
+		dev->stat.scompleted++;
+		spin_unlock_irq(&dev->stat_lock);
+
+		crypto_device_put(dev);
+	}
+}
+
+void crypto_stat_ptime_inc(struct crypto_session *s)
+{
+	struct crypto_device *dev;
+
+	dev = crypto_route_get_current_device(s);
+	if (dev) {
+		int i;
+
+		spin_lock_irq(&dev->stat_lock);
+		for (i = 0; i < dev->cap_number; ++i) {
+			if (__match_initializer(&dev->cap[i], &s->ci)) {
+				dev->cap[i].ptime += s->ci.ptime;
+				dev->cap[i].scomp++;
+				break;
+			}
+		}
+		spin_unlock_irq(&dev->stat_lock);
+
+		crypto_device_put(dev);
+	}
+}
+
+EXPORT_SYMBOL(crypto_stat_start_inc);
+EXPORT_SYMBOL(crypto_stat_finish_inc);
+EXPORT_SYMBOL(crypto_stat_complete_inc);

@@ -0,0 +1,32 @@
+/*
+ * 	crypto_stat.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_STAT_H
+#define __CRYPTO_STAT_H
+
+#include "acrypto.h"
+
+void crypto_stat_start_inc(struct crypto_session *s);
+void crypto_stat_finish_inc(struct crypto_session *s);
+void crypto_stat_complete_inc(struct crypto_session *s);
+void crypto_stat_ptime_inc(struct crypto_session *s);
+
+#endif				/* __CRYPTO_STAT_H */

@@ -0,0 +1,196 @@
+/*
+ * 	crypto_user.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/vmalloc.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+
+#include "acrypto.h"
+#include "crypto_user.h"
+
+static struct scatterlist *crypto_user_alloc_sg(int sg_num)
+{
+	struct scatterlist *sg;
+
+	sg = kmalloc(sg_num * sizeof(*sg), GFP_ATOMIC);
+	if (!sg) {
+		dprintk("Failed to allocate %d scatterlist structures.\n", sg_num);
+		return NULL;
+	}
+
+	memset(sg, 0, sizeof(*sg) * sg_num);
+
+	return sg;
+}
+
+static void crypto_user_free_sg(struct scatterlist *sg)
+{
+	kfree(sg);
+}
+
+int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size)
+{
+	int sg_num;
+
+	if (	(src_size > MAX_DATA_SIZE * PAGE_SIZE) ||
+		(dst_size > MAX_DATA_SIZE * PAGE_SIZE) ||
+		(key_size > MAX_DATA_SIZE * PAGE_SIZE) ||
+		(iv_size  > MAX_DATA_SIZE * PAGE_SIZE)) {
+		dprintk("Sizes are too big: src=%u, dst=%u, key=%u, iv=%u, max=%u.\n",
+				src_size, dst_size, key_size, iv_size, MAX_DATA_SIZE);
+		return -EINVAL;
+	
+	}
+
+	sg_num = ALIGN_DATA_SIZE(src_size) / PAGE_SIZE;
+	data->sg_src = crypto_user_alloc_sg(sg_num);
+	if (!data->sg_src)
+		goto err_out_exit;
+	data->sg_src_num = sg_num;
+	
+	sg_num = ALIGN_DATA_SIZE(dst_size) / PAGE_SIZE;
+	data->sg_dst = crypto_user_alloc_sg(sg_num);
+	if (!data->sg_dst)
+		goto err_out_free_src;
+	data->sg_dst_num = sg_num;
+	
+	sg_num = ALIGN_DATA_SIZE(key_size) / PAGE_SIZE;
+	data->sg_key = crypto_user_alloc_sg(sg_num);
+	if (!data->sg_key)
+		goto err_out_free_dst;
+	data->sg_key_num = sg_num;
+	
+	sg_num = ALIGN_DATA_SIZE(iv_size) / PAGE_SIZE;
+	data->sg_iv = crypto_user_alloc_sg(sg_num);
+	if (!data->sg_iv)
+		goto err_out_free_key;
+	data->sg_iv_num = sg_num;
+
+	return 0;
+
+err_out_free_key:
+	crypto_user_free_sg(data->sg_key);
+err_out_free_dst:
+	crypto_user_free_sg(data->sg_dst);
+err_out_free_src:
+	crypto_user_free_sg(data->sg_src);
+err_out_exit:
+
+	return -ENOMEM;
+}
+
+void crypto_user_free_crypto_data(struct crypto_data *data)
+{
+	crypto_user_free_sg(data->sg_src);
+	crypto_user_free_sg(data->sg_dst);
+	crypto_user_free_sg(data->sg_key);
+	crypto_user_free_sg(data->sg_iv);
+}
+
+void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg)
+{
+	int i, sg_num;
+
+	sg_num = ALIGN_DATA_SIZE(size) / PAGE_SIZE;
+
+	dprintk("Filling %d sgs, total size %u: ", sg_num, size);
+	
+	for (i=0; i<sg_num; ++i) {
+		sg[i].page = virt_to_page(ptr);
+		if (i == 0) {
+			sg[i].offset = offset_in_page(ptr);
+			sg[i].length = ALIGN_DATA_SIZE((unsigned long)ptr) - (unsigned long)ptr;
+			if (sg[i].length == 0)
+				sg[i].length = PAGE_SIZE;
+			if (sg[i].length > size) 
+				sg[i].length = size;
+		} else {
+			sg[i].offset = 0;
+			sg[i].length = (i != sg_num-1)?PAGE_SIZE:size;
+		}
+		dprintka("%x.%x.%p.%lx ", sg[i].offset, sg[i].length, ptr, ALIGN_DATA_SIZE((unsigned long)ptr));
+		
+		size 	-= sg[i].length;
+		ptr 	+= sg[i].length;
+	}
+	dprintka("\n");
+}
+
+struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data)
+{
+	struct scatterlist *sg = NULL;
+	int inval = 0;
+	
+	switch (ud->data_type) {
+		case CRYPTO_USER_DATA_SRC:
+			sg = data->sg_src;
+			inval = (data->sg_src_num * PAGE_SIZE < ud->data_size);
+			dprintk("Found SRC data type, inval=%d, size=%u.\n", inval, ud->data_size);
+			break;
+		case CRYPTO_USER_DATA_DST:
+			sg = data->sg_dst;
+			inval = (data->sg_dst_num * PAGE_SIZE < ud->data_size);
+			dprintk("Found DST data type, inval=%d, size=%u.\n", inval, ud->data_size);
+			break;
+		case CRYPTO_USER_DATA_KEY:
+			sg = data->sg_key;
+			inval = (data->sg_key_num * PAGE_SIZE < ud->data_size);
+			dprintk("Found KEY data type, inval=%d, size=%u.\n", inval, ud->data_size);
+			break;
+		case CRYPTO_USER_DATA_IV:
+			sg = data->sg_iv;
+			inval = (data->sg_iv_num * PAGE_SIZE < ud->data_size);
+			dprintk("Found  IV data type, inval=%d, size=%u.\n", inval, ud->data_size);
+			break;
+		default:
+			dprintk("Unknown data type 0x%x, size=%u.\n", ud->data_type, ud->data_size);
+			break;
+	}
+
+	return (inval)?NULL:sg;
+}
+
+int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr)
+{
+	struct scatterlist *sg;
+
+	sg = crypto_user_get_sg(ud, data);
+	if (!sg)
+		return -EINVAL;
+
+	crypto_user_fill_sg(ptr, ud->data_size, sg);
+
+	return 0;
+}
+
+EXPORT_SYMBOL_GPL(crypto_user_alloc_crypto_data);
+EXPORT_SYMBOL_GPL(crypto_user_free_crypto_data);
+EXPORT_SYMBOL_GPL(crypto_user_fill_sg);
+EXPORT_SYMBOL_GPL(crypto_user_fill_sg_data);
+EXPORT_SYMBOL_GPL(crypto_user_get_sg);

@@ -0,0 +1,52 @@
+/*
+ * 	crypto_user.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_USER_H
+#define __CRYPTO_USER_H
+
+#define MAX_DATA_SIZE	3
+#define ALIGN_DATA_SIZE(size)	((size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))
+
+enum crypto_user_data_types 
+{
+	CRYPTO_USER_DATA_SRC = 0,
+	CRYPTO_USER_DATA_DST,
+	CRYPTO_USER_DATA_KEY,
+	CRYPTO_USER_DATA_IV,
+};
+
+struct crypto_user_data
+{
+	__u16		data_size;
+	__u16		data_type;
+};
+
+
+#ifdef __KERNEL__
+
+int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size);
+void crypto_user_free_crypto_data(struct crypto_data *data);
+void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg);
+struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data);
+int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr);
+
+#endif /* __KERNEL__ */
+#endif /* __CRYPTO_USER_H */

@@ -0,0 +1,390 @@
+/*
+ * 	crypto_user_direct.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/vmalloc.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/connector.h>
+
+#include "acrypto.h"
+#include "crypto_user.h"
+#include "crypto_user_direct.h"
+
+extern struct cb_id crypto_conn_id;
+
+static LIST_HEAD(crypto_user_direct_list);
+static spinlock_t crypto_user_direct_lock = SPIN_LOCK_UNLOCKED;
+static struct completion thread_exited;
+static int need_exit;
+static DECLARE_WAIT_QUEUE_HEAD(crypto_user_direct_wait_queue);
+
+static int crypto_user_direct_alloc_pages(struct crypto_user_direct_kern *k)
+{
+	k->sp = kmalloc(sizeof(struct page *) * k->snum, GFP_KERNEL);
+	if (!k->sp) {
+		dprintk("Failed to allocate %d source pages.\n", k->snum);
+		return -ENOMEM;
+	}
+	
+	k->dp = kmalloc(sizeof(struct page *) * k->dnum, GFP_KERNEL);
+	if (!k->dp) {
+		dprintk("Failed to allocate %d destination pages.\n", k->dnum);
+		kfree(k->sp);
+		return -ENOMEM;
+	}
+	
+	return 0;
+}
+
+static void crypto_user_direct_free_pages(struct crypto_user_direct_kern *k)
+{
+	kfree(k->sp);
+	kfree(k->dp);
+}
+
+static int crypto_user_direct_alloc_vmas(struct crypto_user_direct_kern *k)
+{
+	k->svma = kmalloc(sizeof(struct vm_area_struct *) * k->snum, GFP_KERNEL);
+	if (!k->svma) {
+		dprintk("Failed to allocate %d source vmas.\n", k->snum);
+		return -ENOMEM;
+	}
+	
+	k->dvma = kmalloc(sizeof(struct vm_area_struct *) * k->dnum, GFP_KERNEL);
+	if (!k->dvma) {
+		dprintk("Failed to allocate %d destination vmas.\n", k->dnum);
+		kfree(k->svma);
+		return -ENOMEM;
+	}
+	
+	return 0;
+}
+
+static void crypto_user_direct_free_vmas(struct crypto_user_direct_kern *k)
+{
+	kfree(k->svma);
+	kfree(k->dvma);
+}
+
+static int crypto_user_direct_alloc_mm(struct crypto_user_direct_kern *k)
+{
+	int err;
+
+	err = crypto_user_direct_alloc_pages(k);
+	if (err)
+		return err;
+	
+	err = crypto_user_direct_alloc_vmas(k);
+	if (err) {
+		crypto_user_direct_free_pages(k);
+		return err;
+	}
+
+	return 0;
+}
+
+static void crypto_user_direct_free_mm(struct crypto_user_direct_kern *k)
+{
+	crypto_user_direct_free_pages(k);
+	crypto_user_direct_free_vmas(k);
+}
+
+static void crypto_user_direct_free_data(struct crypto_user_direct_kern *k)
+{
+	int i;
+	
+	for (i=0; i<k->snum; ++i)
+		page_cache_release(k->sp[i]);
+	for (i=0; i<k->dnum; ++i) {
+		set_page_dirty_lock(k->dp[i]);
+		page_cache_release(k->dp[i]);
+	}
+	up_read(&k->mm->mmap_sem);
+	crypto_user_direct_free_mm(k);
+	mmput(k->mm);
+}
+
+static void crypto_user_direct_callback(struct crypto_session_initializer *ci, struct crypto_data *data)
+{
+	struct crypto_user_direct_kern *k = data->priv;
+	struct cn_msg m;
+
+	memset(&m, 0, sizeof(m));
+
+	memcpy(&m.id, &crypto_conn_id, sizeof(m.id));
+	m.seq = k->seq;
+	m.ack = k->ack+1;
+
+	cn_netlink_send(&m, 0);
+
+	crypto_user_direct_free_data(k);
+	crypto_user_free_crypto_data(data);
+}
+
+static void crypto_user_direct_fill_data(struct crypto_data *data, struct crypto_user_direct_kern *k)
+{
+	int i, size;
+
+	size = k->usr.src_size;
+	for (i=0; i<data->sg_src_num; ++i) {
+		data->sg_src[i].page = k->sp[i];
+
+		if (i == 0) {
+			data->sg_src[i].offset = offset_in_page(k->usr.src);
+			data->sg_src[i].length = ALIGN_DATA_SIZE(k->usr.src) - k->usr.src;
+		} else {
+			data->sg_src[i].offset = 0;
+			data->sg_src[i].length = (i != data->sg_src_num)?PAGE_SIZE:size;
+		}
+
+		size -= data->sg_src[i].length;
+	}
+	
+	size = k->usr.dst_size;
+	for (i=0; i<data->sg_dst_num; ++i) {
+		data->sg_dst[i].page = k->dp[i];
+
+		if (i == 0) {
+			data->sg_dst[i].offset = offset_in_page(k->usr.dst);
+			data->sg_dst[i].length = ALIGN_DATA_SIZE(k->usr.dst) - k->usr.dst;
+		} else {
+			data->sg_dst[i].offset = 0;
+			data->sg_dst[i].length = (i != data->sg_dst_num)?PAGE_SIZE:size;
+		}
+
+		size -= data->sg_dst[i].length;
+	}
+}
+
+static int crypto_user_direct_prepare_data(struct crypto_data *data, struct crypto_user_direct_kern *k)
+{
+	int err, i;
+	struct task_struct *tsk;
+
+	tsk = find_task_by_pid(k->usr.pid);
+	if (!tsk) {
+		dprintk(KERN_ERR "Task with pid=%d does not exist.\n", k->usr.pid);
+		return -ENODEV;
+	}
+
+	dprintk("Found task with pid=%d.\n", k->usr.pid);
+
+	k->mm = get_task_mm(tsk);
+	if (!k->mm)
+		return -EINVAL;
+
+	k->snum = data->sg_src_num;
+	k->dnum = data->sg_dst_num;
+
+	err = crypto_user_direct_alloc_mm(k);
+	if (err)
+		goto err_out_put_mm;
+
+	down_read(&k->mm->mmap_sem);
+		
+	err = get_user_pages(tsk, k->mm, k->usr.src, k->snum, 1, 1, k->sp, k->svma);
+	if (err < 0) {
+		dprintk("Failed to get %d src pages for pid=%d.\n",
+				k->snum, k->usr.pid);
+		goto err_out_up_sem;
+	}
+	
+	err = get_user_pages(tsk, k->mm, k->usr.dst, k->dnum, 1, 1, k->dp, k->dvma);
+	if (err < 0) {
+		dprintk("Failed to get %d dst pages for pid=%d.\n",
+				k->snum, k->usr.pid);
+		goto err_out_put_src;
+	}
+
+	crypto_user_direct_fill_data(data, k);
+
+	return 0;
+
+err_out_put_src:
+	for (i=0; i<k->snum; ++i)
+		page_cache_release(k->sp[i]);
+err_out_up_sem:
+	up_read(&k->mm->mmap_sem);
+	
+	crypto_user_direct_free_mm(k);
+err_out_put_mm:
+	mmput(k->mm);
+	return err;
+}
+
+static int crypto_user_direct_prepare(struct crypto_session_initializer *ci, struct crypto_data *data, struct crypto_user_direct_kern *k)
+{
+	int err;
+
+	ci->operation 		= k->usr.operation;
+	ci->type 		= k->usr.type;
+	ci->mode 		= k->usr.mode;
+	ci->priority 		= k->usr.priority;
+	ci->callback 		= crypto_user_direct_callback;
+
+	err = crypto_user_alloc_crypto_data(data, k->usr.src_size, k->usr.dst_size, k->usr.key_size, k->usr.iv_size);
+	if (err)
+		return err;
+
+	if (k->usr.key_size)
+		crypto_user_fill_sg(k->key, k->usr.key_size, data->sg_key);
+	
+	if (k->usr.iv_size)
+		crypto_user_fill_sg(k->iv, k->usr.iv_size, data->sg_iv);
+
+	data->priv		= k;
+	data->priv_size		= 0;
+
+	err = crypto_user_direct_prepare_data(data, k);
+	if (err) {
+		crypto_user_free_crypto_data(data);
+		return err;
+	}
+
+	return 0;
+}
+
+static int crypto_user_direct_process(struct crypto_user_direct_kern *k)
+{
+	struct crypto_session_initializer ci;
+	struct crypto_data data;
+	struct crypto_session *s;
+	int err;
+
+	memset(&ci, 0, sizeof(ci));
+	memset(&data, 0, sizeof(data));
+
+	err = crypto_user_direct_prepare(&ci, &data, k);
+	if (err)
+		return err;
+
+	s = crypto_session_alloc(&ci, &data);
+	if (!s) {
+		crypto_user_direct_free_data(k);
+		return -EINVAL;
+	}
+
+	return 0;
+}	
+
+static int crypto_user_direct_thread(void *data)
+{
+	struct crypto_user_direct_kern *k, *n;
+
+	daemonize("crypto_user_direct_thread");
+	allow_signal(SIGTERM);
+
+	while (!need_exit) {
+		interruptible_sleep_on_timeout(&crypto_user_direct_wait_queue, 1000);
+
+		spin_lock_bh(&crypto_user_direct_lock);
+		list_for_each_entry_safe(k, n, &crypto_user_direct_list, entry) {
+			list_del(&k->entry);
+
+			spin_unlock_bh(&crypto_user_direct_lock);
+			
+			crypto_user_direct_process(k);
+			
+			spin_lock_bh(&crypto_user_direct_lock);
+		}
+		spin_unlock_bh(&crypto_user_direct_lock);
+	}
+
+	complete_and_exit(&thread_exited, 0);
+}
+
+int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr)
+{
+	struct crypto_user_direct_kern *k;
+
+	k = kmalloc(sizeof(struct crypto_user_direct_kern) + usr->key_size + usr->iv_size, GFP_ATOMIC);
+	if (!k) {
+		dprintk("Failed to allocate new kernel crypto request.\n");
+		return -ENOMEM;
+	}
+
+	memset(k, 0, sizeof(*k));
+	
+	memcpy(&k->usr, usr, sizeof(k->usr));
+	
+	k->key = (u8 *)(k+1);
+	k->iv = (u8 *)(k->key + k->usr.key_size);
+
+	memcpy(k->key, usr->data, k->usr.key_size);
+	memcpy(k->iv, usr->data + k->usr.key_size, k->usr.iv_size);
+
+	k->seq = seq;
+	k->ack = ack;
+		
+	spin_lock_bh(&crypto_user_direct_lock);
+	list_add(&k->entry, &crypto_user_direct_list);
+	spin_unlock_bh(&crypto_user_direct_lock);
+
+	dprintk("msg [%08x.%08x]: op=[%04x.%04x.%04x.%04x], src=%llx [%d], dst=%llx [%d], key=%p [%d], iv=%p [%d].\n",
+			seq, ack, 
+			k->usr.operation, k->usr.mode, k->usr.type, k->usr.priority,
+			k->usr.src, k->usr.src_size,
+			k->usr.dst, k->usr.dst_size,
+			k->key, k->usr.key_size,
+			k->iv, k->usr.iv_size);
+
+	wake_up_interruptible(&crypto_user_direct_wait_queue);
+
+	return 0;
+}
+
+int crypto_user_direct_init(void)
+{
+	int pid, err;
+
+	err = 0;
+	init_completion(&thread_exited);
+	pid = kernel_thread(crypto_user_direct_thread, NULL, CLONE_FS | CLONE_FILES);
+	if (IS_ERR((void *)pid)) {
+		dprintk(KERN_ERR "Failed to create acrypto userspace processing thread.\n");
+		err = -EINVAL;
+		goto err_out_exit;
+	}
+
+	printk(KERN_INFO "Acrypto userspace processing thread has been started.\n");
+
+	return err;
+
+err_out_exit:
+
+	return err;
+}
+
+void crypto_user_direct_fini(void)
+{
+	need_exit = 1;
+	wait_for_completion(&thread_exited);
+	
+	printk(KERN_INFO "Acrypto userspace processing thread has been finished.\n");
+}

@@ -0,0 +1,74 @@
+/*
+ * 	crypto_user_direct.h
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef __CRYPTO_USER_DIRECT_H
+#define __CRYPTO_USER_DIRECT_H
+
+struct crypto_user_direct
+{
+	__u64			src;
+	__u32			src_size;
+	__u64			dst;
+	__u32			dst_size;
+	
+	__u16 			operation;
+	__u16 			type;
+	__u16 			mode;
+	__u16 			priority;
+
+	int			pid;
+
+	int			key_size;
+	int			iv_size;
+
+	__u8			data[0];
+};
+
+#ifdef __KERNEL__
+
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+
+struct crypto_user_direct_kern
+{
+	struct list_head	entry;
+
+	u32			seq;
+	u32			ack;
+
+	struct crypto_user_direct	usr;
+	u8			*key;
+	u8			*iv;
+
+	int			snum, dnum;
+	struct page		**sp, **dp;
+	struct vm_area_struct 	**svma, **dvma;
+	struct mm_struct 	*mm;
+};
+
+int crypto_user_direct_init(void);
+void crypto_user_direct_fini(void);
+int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr);
+
+#endif /* __KERNEL__ */
+
+#endif /* __CRYPTO_USER_DIRECT_H */

@@ -0,0 +1,281 @@
+/*
+ * 	crypto_user_ioctl.c
+ *
+ * Copyright (c) 2004 Evgeniy Polyakov <[email protected]>
+ * 
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/fs.h>
+
+#include <asm/uaccess.h>
+
+#include "acrypto.h"
+#include "crypto_user.h"
+#include "crypto_user_ioctl.h"
+
+static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg);
+static int crypto_user_ioctl_open(struct inode *inode, struct file *fp);
+static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp);
+int crypto_user_ioctl_init(void);
+void crypto_user_ioctl_fini(void);
+
+static int crypto_user_ioctl_major = 0;
+static char crypto_user_ioctl_name[] = "crypto_user_ioctl";
+
+static struct file_operations crypto_user_ioctl_ops = {
+	.open		crypto_user_ioctl_open,
+	.release	crypto_user_ioctl_release,
+	.ioctl		crypto_user_ioctl_ioctl,
+	.owner 		THIS_MODULE
+};
+
+static void dump_data(u8 *ptr)
+{
+	int i;
+
+	dprintk("USER DATA: ");
+	for (i=0; i<32; ++i)
+		dprintka("%02x ", ptr[i]);
+	dprintka("\n");
+}
+
+static int crypto_user_ioctl_open(struct inode *inode, struct file *fp)
+{
+	struct crypto_user_ioctl_kern *iok;
+
+	iok = kmalloc(sizeof(*iok), GFP_KERNEL);
+	if (!iok) {
+		dprintk("Failed to allocate new crypto_user_ioctl_kern structure.\n");
+		return -ENOMEM;
+	}
+	memset(iok, 0, sizeof(*iok));
+
+	fp->private_data = iok;
+
+	return 0;
+}
+
+static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp)
+{
+	struct crypto_user_ioctl_kern *iok = fp->private_data;
+	int i;
+
+	for (i=0; i<4; ++i)
+		if (iok->ptr[i])
+			kfree(iok->ptr[i]);
+	kfree(iok);
+
+	return 0;
+}
+
+static void crypto_user_ioctl_callback(struct crypto_session_initializer *ci, struct crypto_data *data)
+{
+	struct crypto_user_ioctl_kern *iok = data->priv;
+	
+	dprintk("%s() for session %llu [%llu].\n",
+			__func__, iok->s->ci.id, iok->s->ci.dev_id);
+			
+	crypto_user_free_crypto_data(&iok->data);
+
+	iok->scompleted = 1;
+	wake_up_interruptible(&iok->wait);
+}
+
+static int crypto_user_ioctl_session_alloc(struct crypto_user_ioctl *io, struct crypto_user_ioctl_kern *iok)
+{
+	int err;
+	
+	err = crypto_user_alloc_crypto_data(&iok->data, io->src_size, io->dst_size, io->key_size, io->iv_size);
+	if (err)
+		return err;
+
+	iok->ci.operation 	= io->operation;
+	iok->ci.type 		= io->type;
+	iok->ci.mode 		= io->mode;
+	iok->ci.priority 	= io->priority;
+	iok->ci.callback 	= crypto_user_ioctl_callback;
+
+	iok->data.priv		= iok;
+	iok->data.priv_size	= 0;
+	
+	iok->scompleted 	= 0;
+
+	init_waitqueue_head(&iok->wait);
+	
+	iok->s = crypto_session_create(&iok->ci, &iok->data);
+	if (!iok->s) {
+		crypto_user_free_crypto_data(&iok->data);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int crypto_user_ioctl_session_add(struct crypto_user_ioctl_kern *iok)
+{
+	crypto_session_add(iok->s);
+
+	return 0;
+}
+
+static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg)
+{
+	struct crypto_user_ioctl io;
+	struct crypto_user_data data;
+	unsigned long not_read;
+	int err;
+	struct crypto_user_ioctl_kern *iok;
+       
+	iok = fp->private_data;
+	
+	err = 0;
+	switch (cmd) {
+		case CRYPTO_SESSION_ALLOC:
+			not_read = copy_from_user(&io, (void __user *)arg, sizeof(io));
+			if (not_read) {
+				dprintk("Failed to read crypto_user_ioctl structure from userspace.\n");
+				err = -EINVAL;
+				break;
+			}
+
+			err = crypto_user_ioctl_session_alloc(&io, iok);
+			break;
+		case CRYPTO_FILL_DATA:
+			not_read = copy_from_user(&data, (void __user *)arg, sizeof(data));
+			if (not_read) {
+				dprintk("Failed to read crypto_user_ioctl_data structure from userspace.\n");
+				err = -EINVAL;
+				break;
+			}
+
+			if (data.data_size > MAX_DATA_SIZE * PAGE_SIZE) {
+				dprintk("Data size is too bit: size=%u, type=%x.\n",
+						data.data_size, data.data_type);
+				err = -EINVAL;
+				break;
+			}
+
+			if (!crypto_user_get_sg(&data, &iok->data)) {
+				dprintk("Invalid crypto_user_data structure [size=%u, type=%x].\n", 
+						data.data_size, data.data_type);
+				err = -EINVAL;
+				break;
+			}
+
+			if (iok->ptr[data.data_type])
+				kfree(iok->ptr[data.data_type]);
+
+			iok->ptr[data.data_type] = kmalloc(data.data_size, GFP_KERNEL);
+			if (!iok->ptr[data.data_type]) {
+				dprintk("Failed to allocate %d bytes for data type %d.\n",
+						data.data_size, data.data_type);
+				err = -ENOMEM;
+				break;
+			}
+
+			not_read = copy_from_user(iok->ptr[data.data_type], (void __user *)arg + sizeof(data), data.data_size);
+			if (not_read) {
+				dprintk("Failed to read %d bytes of crypto data [type=%d] from userspace.\n",
+						data.data_size, data.data_type);
+				kfree(iok->ptr[data.data_type]);
+				err = -EINVAL;
+				break;
+			}
+
+			memcpy(&iok->usr[data.data_type], &data, sizeof(struct crypto_user_data));
+			
+			err = crypto_user_fill_sg_data(&data, &iok->data, iok->ptr[data.data_type]);
+			if (err) {
+				kfree(iok->ptr[data.data_type]);
+				break;
+			}
+			break;
+		case CRYPTO_SESSION_ADD:
+			if (!iok->s) {
+				dprintk("CRYPTO_SESSION_ADD must be called after session initialisation.\n");
+				err = -EINVAL;
+				break;
+			}
+			
+			err = crypto_user_ioctl_session_add(iok);
+			if (err)
+				break;
+
+			wait_event_interruptible(iok->wait, iok->scompleted);
+
+			dump_data(iok->ptr[CRYPTO_USER_DATA_DST]);
+
+			not_read = copy_to_user((void __user *)arg, iok->ptr[CRYPTO_USER_DATA_DST], iok->usr[CRYPTO_USER_DATA_DST].data_size);
+			if (not_read) {
+				dprintk("Failed to copy to user %d bytes of result.\n", iok->usr[CRYPTO_USER_DATA_DST].data_size);
+				err = -EINVAL;
+				break;
+			}
+			break;
+
+		default:
+			dprintk("Invalid ioctl(0x%x).\n", cmd);
+			err = -ENODEV;
+			break;
+	}
+	
+	return err;
+}
+
+static ssize_t crypto_user_ioctl_dev_show(struct class_device *dev, char *buf)
+{
+	return sprintf(buf, "%u:%u\n", crypto_user_ioctl_major, 0);
+}
+
+extern struct crypto_device main_crypto_device;
+static CLASS_DEVICE_ATTR(dev, 0644, crypto_user_ioctl_dev_show, NULL);
+
+int crypto_user_ioctl_init(void)
+{
+	struct crypto_device *dev = &main_crypto_device;
+	int err;
+
+	crypto_user_ioctl_major = register_chrdev(0, crypto_user_ioctl_name, &crypto_user_ioctl_ops);
+       	if (crypto_user_ioctl_major < 0) {
+		dprintk("Failed to register %s char device: err=%d.\n", crypto_user_ioctl_name, crypto_user_ioctl_major);
+		return -ENODEV;
+	};
+	
+	err = class_device_create_file(&dev->class_device, &class_device_attr_dev);
+	if (err)
+		dprintk("Failed to create \"dev\" attribute: err=%d.\n", err);
+
+	printk("Asynchronous crypto userspace helper(ioctl based) has been started, major=%d.\n", crypto_user_ioctl_major);
+
+	return 0;
+}
+
+void crypto_user_ioctl_fini(void)
+{
+	struct crypto_device *dev = &main_crypto_device;
+
+	class_device_remove_file(&dev->class_device, &class_device_attr_dev);
+	unregister_chrdev(crypto_user_ioctl_major, crypto_user_ioctl_name);
+}