Parallel IPsec v3

index 91bb76f..a17679c 100644
@@ -338,7 +338,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
-	RCU_SOFTIRQ,	/* Preferable RCU should always be the last softirq */
+	PADATA_SOFTIRQ,
+	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
 };

116 lines not on this page · raw message

490 lines not on this page · raw message

index b525dd3..12c9b64 100644
@@ -56,7 +56,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER",	"RCU"
+	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
 };
 
 /*

index d9aa733..f45e27a 100644
@@ -286,6 +286,7 @@ struct crypto_instance *aead_geniv_alloc(struct crypto_template *tmpl,
 	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
 	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
 	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+	inst->alg.cra_aead.wrapper = alg->cra_aead.wrapper;
 
 	inst->alg.cra_aead.setkey = alg->cra_aead.setkey;
 	inst->alg.cra_aead.setauthsize = alg->cra_aead.setauthsize;

@@ -407,6 +408,262 @@ out:
 	return err;
 }
 
+static int crypto_init_wrapaead_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
+{
+	struct aead_alg *alg = &tfm->__crt_alg->cra_aead;
+	struct aead_tfm *crt = &tfm->crt_aead;
+
+	if (max(alg->maxauthsize, alg->ivsize) > PAGE_SIZE / 8)
+		return -EINVAL;
+
+	crt->setkey = alg->setkey;
+	crt->encrypt = alg->encrypt;
+	crt->decrypt = alg->decrypt;
+	crt->givencrypt = alg->givencrypt ?: no_givcrypt;
+	crt->givdecrypt = alg->givdecrypt ?: no_givcrypt;
+	crt->base = __crypto_aead_cast(tfm);
+	crt->ivsize = alg->ivsize;
+	crt->authsize = alg->maxauthsize;
+
+	return 0;
+}
+
+static void crypto_wrapaead_show(struct seq_file *m, struct crypto_alg *alg)
+	__attribute__ ((unused));
+static void crypto_wrapaead_show(struct seq_file *m, struct crypto_alg *alg)
+{
+	struct aead_alg *aead = &alg->cra_aead;
+
+	seq_printf(m, "type         : wrapped aead\n");
+	seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
+					     "yes" : "no");
+	seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
+	seq_printf(m, "ivsize       : %u\n", aead->ivsize);
+	seq_printf(m, "maxauthsize  : %u\n", aead->maxauthsize);
+	seq_printf(m, "geniv        : %s\n", aead->geniv ?: "<built-in>");
+	seq_printf(m, "wrapper      : %s\n", alg->cra_flags & CRYPTO_ALG_WRAP ?
+					     aead->wrapper : "<built-in>");
+}
+
+const struct crypto_type crypto_wrapaead_type = {
+	.ctxsize = crypto_aead_ctxsize,
+	.init = crypto_init_wrapaead_ops,
+#ifdef CONFIG_PROC_FS
+	.show = crypto_wrapaead_show,
+#endif
+};
+EXPORT_SYMBOL_GPL(crypto_wrapaead_type);
+
+static int crypto_grab_wrapaead(struct crypto_aead_spawn *spawn,
+			       const char *name, u32 type, u32 mask)
+{
+	struct crypto_alg *alg;
+	int err;
+
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP);
+	type |= CRYPTO_ALG_TYPE_AEAD;
+	mask |= CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP;
+
+	alg = crypto_alg_mod_lookup(name, type, mask);
+	if (IS_ERR(alg))
+		return PTR_ERR(alg);
+
+	err = crypto_init_spawn(&spawn->base, alg, spawn->base.inst, mask);
+	crypto_mod_put(alg);
+	return err;
+}
+
+struct crypto_instance *aead_wrap_alloc(struct crypto_template *tmpl,
+					 struct rtattr **tb,
+					 unsigned int ctx_size, u32 type,
+					 u32 mask)
+{
+	const char *name;
+	struct crypto_aead_spawn *spawn;
+	struct crypto_attr_type *algt;
+	struct crypto_instance *inst;
+	struct crypto_alg *alg;
+	int err;
+
+	algt = crypto_get_attr_type(tb);
+	err = PTR_ERR(algt);
+	if (IS_ERR(algt))
+		return ERR_PTR(err);
+
+	if ((algt->type ^ (CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP)) &
+	    algt->mask)
+		return ERR_PTR(-EINVAL);
+
+	name = crypto_attr_alg_name(tb[1]);
+	err = PTR_ERR(name);
+	if (IS_ERR(name))
+		return ERR_PTR(err);
+
+	inst = kzalloc(sizeof(*inst) + ctx_size, GFP_KERNEL);
+	if (!inst)
+		return ERR_PTR(-ENOMEM);
+
+	spawn = crypto_instance_ctx(inst);
+
+	/* Ignore async algorithms if necessary. */
+	mask |= crypto_requires_sync(algt->type, algt->mask);
+
+	crypto_set_aead_spawn(spawn, inst);
+	err = crypto_grab_wrapaead(spawn, name, type, mask);
+	if (err)
+		goto err_free_inst;
+
+	alg = crypto_aead_spawn_alg(spawn);
+
+	err = -EINVAL;
+	if (alg->cra_type != &crypto_aead_type)
+		goto err_drop_alg;
+
+	if (algt->mask & CRYPTO_ALG_WRAP) {
+		if (strcmp(tmpl->name, alg->cra_aead.wrapper))
+			goto err_drop_alg;
+
+		memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
+		memcpy(inst->alg.cra_driver_name, alg->cra_driver_name,
+		       CRYPTO_MAX_ALG_NAME);
+	} else {
+		err = -ENAMETOOLONG;
+		if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
+			     "%s(%s)", tmpl->name, alg->cra_name) >=
+		    CRYPTO_MAX_ALG_NAME)
+			goto err_drop_alg;
+		if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+			     "%s(%s)", tmpl->name, alg->cra_driver_name) >=
+		    CRYPTO_MAX_ALG_NAME)
+			goto err_drop_alg;
+	}
+
+	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP;
+	inst->alg.cra_flags |= alg->cra_flags
+			     & (CRYPTO_ALG_GENIV | CRYPTO_ALG_ASYNC);
+	inst->alg.cra_blocksize = alg->cra_blocksize;
+	inst->alg.cra_alignmask = alg->cra_alignmask;
+	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
+	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
+	inst->alg.cra_type = &crypto_wrapaead_type;
+	inst->alg.cra_aead.wrapper = alg->cra_aead.wrapper;
+	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
+	inst->alg.cra_priority = alg->cra_priority + 100;
+
+out:
+	return inst;
+
+err_drop_alg:
+	crypto_drop_aead(spawn);
+err_free_inst:
+	kfree(inst);
+	inst = ERR_PTR(err);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(aead_wrap_alloc);
+
+void aead_wrap_free(struct crypto_instance *inst)
+{
+	crypto_drop_aead(crypto_instance_ctx(inst));
+	kfree(inst);
+}
+EXPORT_SYMBOL_GPL(aead_wrap_free);
+
+int aead_wrap_init(struct crypto_tfm *tfm)
+{
+	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
+	struct crypto_aead *aead;
+
+	aead = crypto_spawn_aead(crypto_instance_ctx(inst));
+	if (IS_ERR(aead))
+		return PTR_ERR(aead);
+
+	tfm->crt_aead.base = aead;
+	tfm->crt_aead.reqsize += crypto_aead_reqsize(aead);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(aead_wrap_init);
+
+void aead_wrap_exit(struct crypto_tfm *tfm)
+{
+	crypto_free_aead(tfm->crt_aead.base);
+}
+EXPORT_SYMBOL_GPL(aead_wrap_exit);
+
+static int crypto_wrapaead_default(struct crypto_alg *alg, u32 type, u32 mask)
+{
+	struct rtattr *tb[3];
+	struct {
+		struct rtattr attr;
+		struct crypto_attr_type data;
+	} ptype;
+	struct {
+		struct rtattr attr;
+		struct crypto_attr_alg data;
+	} palg;
+	struct crypto_template *tmpl;
+	struct crypto_instance *inst;
+	struct crypto_alg *larval;
+	const char *wrapper;
+	int err;
+
+	larval = crypto_larval_lookup(alg->cra_driver_name,
+				      CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_WRAP,
+				      CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_WRAP);
+	err = PTR_ERR(larval);
+	if (IS_ERR(larval))
+		goto out;
+
+	err = -EAGAIN;
+	if (!crypto_is_larval(larval))
+		goto drop_larval;
+
+	ptype.attr.rta_len = sizeof(ptype);
+	ptype.attr.rta_type = CRYPTOA_TYPE;
+	ptype.data.type = type | CRYPTO_ALG_WRAP;
+	ptype.data.mask = mask | CRYPTO_ALG_WRAP;
+	tb[0] = &ptype.attr;
+
+	palg.attr.rta_len = sizeof(palg);
+	palg.attr.rta_type = CRYPTOA_ALG;
+	/* Must use the exact name to locate ourselves. */
+	memcpy(palg.data.name, alg->cra_driver_name, CRYPTO_MAX_ALG_NAME);
+	tb[1] = &palg.attr;
+
+	tb[2] = NULL;
+
+	wrapper = alg->cra_aead.wrapper;
+
+	tmpl = crypto_lookup_template(wrapper);
+	err = -ENOENT;
+	if (!tmpl)
+		goto kill_larval;
+
+	inst = tmpl->alloc(tb);
+	err = PTR_ERR(inst);
+	if (IS_ERR(inst))
+		goto put_tmpl;
+
+	if ((err = crypto_register_instance(tmpl, inst))) {
+		tmpl->free(inst);
+		goto put_tmpl;
+	}
+
+	/* Redo the lookup to use the instance we just registered. */
+	err = -EAGAIN;
+
+put_tmpl:
+	crypto_tmpl_put(tmpl);
+kill_larval:
+	crypto_larval_kill(larval);
+drop_larval:
+	crypto_mod_put(larval);
+out:
+	crypto_mod_put(alg);
+	return err;
+}
+
 static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 					     u32 mask)
 {

@@ -416,10 +673,13 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 	if (IS_ERR(alg))
 		return alg;
 
-	if (alg->cra_type == &crypto_aead_type)
+	if (alg->cra_type == &crypto_wrapaead_type)
 		return alg;
 
-	if (!alg->cra_aead.ivsize)
+	if ((alg->cra_type == &crypto_aead_type) && !alg->cra_aead.wrapper)
+		return alg;
+
+	if (!alg->cra_aead.ivsize && !alg->cra_aead.wrapper)
 		return alg;
 
 	crypto_mod_put(alg);

@@ -428,7 +688,7 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 	if (IS_ERR(alg))
 		return alg;
 
-	if (alg->cra_type == &crypto_aead_type) {
+	if (alg->cra_type == &crypto_wrapaead_type) {
 		if ((alg->cra_flags ^ type ^ ~mask) & CRYPTO_ALG_TESTED) {
 			crypto_mod_put(alg);
 			alg = ERR_PTR(-ENOENT);

@@ -436,6 +696,18 @@ static struct crypto_alg *crypto_lookup_aead(const char *name, u32 type,
 		return alg;
 	}
 
+	if (alg->cra_type == &crypto_aead_type) {
+		if ((alg->cra_flags ^ type ^ ~mask) & CRYPTO_ALG_TESTED) {
+			crypto_mod_put(alg);
+			alg = ERR_PTR(-ENOENT);
+		}
+
+		if (!alg->cra_aead.wrapper)
+			return alg;
+
+		return ERR_PTR(crypto_wrapaead_default(alg, type, mask));
+	}
+
 	BUG_ON(!alg->cra_aead.ivsize);
 
 	return ERR_PTR(crypto_nivaead_default(alg, type, mask));

@@ -447,9 +719,9 @@ int crypto_grab_aead(struct crypto_aead_spawn *spawn, const char *name,
 	struct crypto_alg *alg;
 	int err;
 
-	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	type |= CRYPTO_ALG_TYPE_AEAD;
-	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	mask |= CRYPTO_ALG_TYPE_MASK;
 
 	alg = crypto_lookup_aead(name, type, mask);

@@ -467,9 +739,9 @@ struct crypto_aead *crypto_alloc_aead(const char *alg_name, u32 type, u32 mask)
 	struct crypto_tfm *tfm;
 	int err;
 
-	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	type |= CRYPTO_ALG_TYPE_AEAD;
-	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
+	mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV | CRYPTO_ALG_WRAP);
 	mask |= CRYPTO_ALG_TYPE_MASK;
 
 	for (;;) {

index d838c94..4bf8a50 100644
@@ -65,6 +65,14 @@ static inline struct crypto_aead *aead_geniv_base(struct crypto_aead *geniv)
 	return crypto_aead_crt(geniv)->base;
 }
 
+struct crypto_instance *aead_wrap_alloc(struct crypto_template *tmpl,
+					 struct rtattr **tb,
+					 unsigned int ctx_size, u32 type,
+					 u32 mask);
+void aead_wrap_free(struct crypto_instance *inst);
+int aead_wrap_init(struct crypto_tfm *tfm);
+void aead_wrap_exit(struct crypto_tfm *tfm);
+
 static inline void *aead_givcrypt_reqctx(struct aead_givcrypt_request *req)
 {
 	return aead_request_ctx(&req->areq);

index ec29fa2..2548bf0 100644
@@ -71,6 +71,8 @@
 
 #define CRYPTO_ALG_TESTED		0x00000400
 
+#define CRYPTO_ALG_WRAP			0x00000800
+
 /*
  * Transform masks and values (for crt_flags).
  */

@@ -242,6 +244,7 @@ struct aead_alg {
 	int (*givdecrypt)(struct aead_givcrypt_request *req);
 
 	const char *geniv;
+	const char *wrapper;
 
 	unsigned int ivsize;
 	unsigned int maxauthsize;

@@ -881,9 +884,12 @@ static inline unsigned int crypto_aead_reqsize(struct crypto_aead *tfm)
 }
 
 static inline void aead_request_set_tfm(struct aead_request *req,
-					struct crypto_aead *tfm)
+					struct crypto_aead *aead)
 {
-	req->base.tfm = crypto_aead_tfm(crypto_aead_crt(tfm)->base);
+	struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+
+	req->base.tfm = tfm->__crt_alg->cra_flags & CRYPTO_ALG_WRAP ?
+			tfm : crypto_aead_tfm(crypto_aead_crt(aead)->base);
 }
 
 static inline struct aead_request *aead_request_alloc(struct crypto_aead *tfm,

index 56c62e2..2492e6c 100644
@@ -255,7 +255,7 @@ found:
 			continue;
 
 		if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
-		    q->cra_priority > alg->cra_priority)
+		    q->cra_priority < alg->cra_priority)
 			continue;
 
 		crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags);

index 2721f07..4aad58f 100644
@@ -344,7 +344,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
-	RCU_SOFTIRQ,	/* Preferable RCU should always be the last softirq */
+	PADATA_SOFTIRQ,
+	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
 };

116 lines not on this page · raw message

490 lines not on this page · raw message

index b41fb71..e0faebf 100644
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER",	"RCU"
+	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
 };
 
 /*

411 lines not on this page · raw message

106 lines not on this page · raw message

51 lines not on this page · raw message

index 4aad58f..67d46e9 100644
@@ -344,6 +344,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
+	AEAD_ENC_SOFTIRQ,
+	AEAD_DEC_SOFTIRQ,
 	PADATA_SOFTIRQ,
 	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 

index 6a98076..feb77e4 100644
@@ -81,16 +81,35 @@ static void crypto_destroy_instance(struct crypto_alg *alg)
 	crypto_tmpl_put(tmpl);
 }
 
+static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
+					    struct list_head *stack,
+					    struct list_head *top,
+					    struct list_head *secondary_spawns)
+{
+	struct crypto_spawn *spawn, *n;
+
+	if (list_empty(stack))
+		return NULL;
+
+	spawn = list_first_entry(stack, struct crypto_spawn, list);
+	n = list_entry(spawn->list.next, struct crypto_spawn, list);
+
+	if (spawn->alg && &n->list != stack && !n->alg)
+		n->alg = (n->list.next == stack) ? alg :
+			 &list_entry(n->list.next, struct crypto_spawn,
+				     list)->inst->alg;
+
+	list_move(&spawn->list, secondary_spawns);
+
+	return &n->list == stack ? top : &n->inst->alg.cra_users;
+}
+
 static void crypto_remove_spawn(struct crypto_spawn *spawn,
-				struct list_head *list,
-				struct list_head *secondary_spawns)
+				struct list_head *list)
 {
 	struct crypto_instance *inst = spawn->inst;
 	struct crypto_template *tmpl = inst->tmpl;
 
-	list_del_init(&spawn->list);
-	spawn->alg = NULL;
-
 	if (crypto_is_dead(&inst->alg))
 		return;
 

@@ -106,25 +125,55 @@ static void crypto_remove_spawn(struct crypto_spawn *spawn,
 	hlist_del(&inst->list);
 	inst->alg.cra_destroy = crypto_destroy_instance;
 
-	list_splice(&inst->alg.cra_users, secondary_spawns);
+	BUG_ON(!list_empty(&inst->alg.cra_users));
 }
 
-static void crypto_remove_spawns(struct list_head *spawns,
-				 struct list_head *list, u32 new_type)
+static void crypto_remove_spawns(struct crypto_alg *alg,
+				 struct list_head *list,
+				 struct crypto_alg *nalg)
 {
+	u32 new_type = (nalg ?: alg)->cra_flags;
 	struct crypto_spawn *spawn, *n;
 	LIST_HEAD(secondary_spawns);
+	struct list_head *spawns;
+	LIST_HEAD(stack);
+	LIST_HEAD(top);
 
+	spawns = &alg->cra_users;
 	list_for_each_entry_safe(spawn, n, spawns, list) {
 		if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
 			continue;
 
-		crypto_remove_spawn(spawn, list, &secondary_spawns);
+		list_move(&spawn->list, &top);
 	}
 
-	while (!list_empty(&secondary_spawns)) {
-		list_for_each_entry_safe(spawn, n, &secondary_spawns, list)
-			crypto_remove_spawn(spawn, list, &secondary_spawns);
+	spawns = ⊤
+	do {
+		while (!list_empty(spawns)) {
+			struct crypto_instance *inst;
+
+			spawn = list_first_entry(spawns, struct crypto_spawn,
+						 list);
+			inst = spawn->inst;
+
+			BUG_ON(&inst->alg == alg);
+
+			list_move(&spawn->list, &stack);
+
+			if (&inst->alg == nalg)
+				break;
+
+			spawn->alg = NULL;
+			spawns = &inst->alg.cra_users;
+		}
+	} while ((spawns = crypto_more_spawns(alg, &stack, &top,
+					      &secondary_spawns)));
+
+	list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
+		if (spawn->alg)
+			list_move(&spawn->list, &spawn->alg->cra_users);
+		else
+			crypto_remove_spawn(spawn, list);
 	}
 }
 

@@ -258,7 +307,7 @@ found:
 		    q->cra_priority > alg->cra_priority)
 			continue;
 
-		crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags);
+		crypto_remove_spawns(q, &list, alg);
 	}
 
 complete:

@@ -330,7 +379,7 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
 
 	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg);
 	list_del_init(&alg->cra_list);
-	crypto_remove_spawns(&alg->cra_users, list, alg->cra_flags);
+	crypto_remove_spawns(alg, list, NULL);
 
 	return 0;
 }

353 lines not on this page · raw message

106 lines not on this page · raw message

56 lines not on this page · raw message

index a17679c..93e0c9f 100644
@@ -338,6 +338,8 @@ enum
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
+	AEAD_ENC_SOFTIRQ,
+	AEAD_DEC_SOFTIRQ,
 	PADATA_SOFTIRQ,
 	RCU_SOFTIRQ, 	/* Preferable RCU should always be the last softirq */
 

index 12c9b64..c76ace0 100644
@@ -56,7 +56,8 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
 	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
-	"TASKLET", "SCHED", "HRTIMER", "PADATA", "RCU"
+	"TASKLET", "SCHED", "HRTIMER", "AEAD_ENC",
+	"AEAD_DEC", "PADATA", "RCU"
 };
 
 /*

index 2a342c8..b6185f4 100644
@@ -15,6 +15,7 @@
  *
  */
 
+#include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/rng.h>
 #include <crypto/scatterwalk.h>

@@ -62,6 +63,29 @@ out:
 	skcipher_givcrypt_complete(req, err);
 }
 
+static void eseqiv_aead_complete2(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+
+	memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
+			 crypto_aead_alignmask(geniv) + 1),
+	       crypto_aead_ivsize(geniv));
+}
+
+static void eseqiv_aead_complete(struct crypto_async_request *base, int err)
+{
+	struct aead_givcrypt_request *req = base->data;
+
+	if (err)
+		goto out;
+
+	eseqiv_aead_complete2(req);
+
+out:
+	aead_givcrypt_complete(req, err);
+}
+
 static void eseqiv_chain(struct scatterlist *head, struct scatterlist *sg,
 			 int chain)
 {

@@ -153,7 +177,93 @@ static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
 	if (err)
 		goto out;
 
-	eseqiv_complete2(req);
+	if (giv != req->giv)
+		eseqiv_complete2(req);
+
+out:
+	return err;
+}
+
+static int eseqiv_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	struct eseqiv_request_ctx *reqctx = aead_givcrypt_reqctx(req);
+	struct aead_request *subreq;
+	crypto_completion_t complete;
+	void *data;
+	struct scatterlist *osrc, *odst;
+	struct scatterlist *dst;
+	struct page *srcp;
+	struct page *dstp;
+	u8 *giv;
+	u8 *vsrc;
+	u8 *vdst;
+	__be64 seq;
+	unsigned int ivsize;
+	unsigned int len;
+	unsigned int flags;
+	int err;
+
+	subreq = (void *)(reqctx->tail + ctx->reqoff);
+	aead_request_set_tfm(subreq, aead_geniv_base(geniv));
+
+	giv = req->giv;
+	complete = req->areq.base.complete;
+	data = req->areq.base.data;
+
+	osrc = req->areq.src;
+	odst = req->areq.dst;
+	srcp = sg_page(osrc);
+	dstp = sg_page(odst);
+	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
+	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
+
+	ivsize = crypto_aead_ivsize(geniv);
+	flags = req->areq.base.flags | CRYPTO_TFM_REQ_SG_HAS_IV;
+
+	if (vsrc != giv + ivsize && vdst != giv + ivsize) {
+		giv = PTR_ALIGN((u8 *)reqctx->tail,
+				crypto_aead_alignmask(geniv) + 1);
+		complete = eseqiv_aead_complete;
+		data = req;
+	}
+
+	aead_request_set_callback(subreq, flags, complete, data);
+
+	sg_init_table(reqctx->src, 2);
+	sg_set_buf(reqctx->src, giv, ivsize);
+	eseqiv_chain(reqctx->src, osrc, vsrc == giv + ivsize);
+
+	dst = reqctx->src;
+	if (osrc != odst) {
+		sg_init_table(reqctx->dst, 2);
+		sg_set_buf(reqctx->dst, giv, ivsize);
+		eseqiv_chain(reqctx->dst, odst, vdst == giv + ivsize);
+
+		dst = reqctx->dst;
+	}
+
+	aead_request_set_crypt(subreq, reqctx->src, dst,
+				     req->areq.cryptlen + ivsize, req->areq.iv);
+	aead_request_set_assoc(subreq, req->areq.assoc, req->areq.assoclen);
+
+	memcpy(req->areq.iv, ctx->salt, ivsize);
+
+	len = ivsize;
+	if (ivsize > sizeof(u64)) {
+		memset(req->giv, 0, ivsize - sizeof(u64));
+		len = sizeof(u64);
+	}
+	seq = cpu_to_be64(req->seq);
+	memcpy(req->giv + ivsize - len, &seq, len);
+
+	err = crypto_aead_encrypt(subreq);
+	if (err)
+		goto out;
+
+	if (giv != req->giv)
+		eseqiv_aead_complete2(req);
 
 out:
 	return err;

@@ -182,6 +292,29 @@ unlock:
 	return eseqiv_givencrypt(req);
 }
 
+static int eseqiv_aead_givencrypt_first(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	int err = 0;
+
+	spin_lock_bh(&ctx->lock);
+	if (crypto_aead_crt(geniv)->givencrypt != eseqiv_aead_givencrypt_first)
+		goto unlock;
+
+	crypto_aead_crt(geniv)->givencrypt = eseqiv_aead_givencrypt;
+	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
+				   crypto_aead_ivsize(geniv));
+
+unlock:
+	spin_unlock_bh(&ctx->lock);
+
+	if (err)
+		return err;
+
+	return eseqiv_aead_givencrypt(req);
+}
+
 static int eseqiv_init(struct crypto_tfm *tfm)
 {
 	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);

@@ -214,20 +347,47 @@ static int eseqiv_init(struct crypto_tfm *tfm)
 	return skcipher_geniv_init(tfm);
 }
 
+static int eseqiv_aead_init(struct crypto_tfm *tfm)
+{
+	struct crypto_aead *geniv = __crypto_aead_cast(tfm);
+	struct eseqiv_ctx *ctx = crypto_aead_ctx(geniv);
+	unsigned long alignmask;
+	unsigned int reqsize;
+
+	spin_lock_init(&ctx->lock);
+
+	alignmask = crypto_tfm_ctx_alignment() - 1;
+	reqsize = sizeof(struct eseqiv_request_ctx);
+
+	if (alignmask & reqsize) {
+		alignmask &= reqsize;
+		alignmask--;
+	}
+
+	alignmask = ~alignmask;
+	alignmask &= crypto_aead_alignmask(geniv);
+
+	reqsize += alignmask;
+	reqsize += crypto_aead_ivsize(geniv);
+	reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
+
+	ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
+
+	tfm->crt_aead.reqsize = reqsize + sizeof(struct aead_request);
+
+	return aead_geniv_init(tfm);
+}
+
 static struct crypto_template eseqiv_tmpl;
 
-static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+static struct crypto_instance *eseqiv_ablkcipher_alloc(struct rtattr **tb)
 {
-	struct crypto_instance *inst;
 	int err;
-
-	err = crypto_get_default_rng();
-	if (err)
-		return ERR_PTR(err);
+	struct crypto_instance *inst;
 
 	inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
-		goto put_rng;
+		goto out;
 
 	err = -EINVAL;
 	if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize)

@@ -235,18 +395,75 @@ static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
 
 	inst->alg.cra_ablkcipher.givencrypt = eseqiv_givencrypt_first;
 
-	inst->alg.cra_init = eseqiv_init;
-	inst->alg.cra_exit = skcipher_geniv_exit;
+	 inst->alg.cra_init = eseqiv_init;
+	 inst->alg.cra_exit = skcipher_geniv_exit;
 
-	inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx);
-	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
+	inst->alg.cra_ctxsize = inst->alg.cra_ablkcipher.ivsize;
 
 out:
 	return inst;
 
 free_inst:
 	skcipher_geniv_free(inst);
-	inst = ERR_PTR(err);
+	return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_aead_alloc(struct rtattr **tb)
+{
+	int err;
+	struct crypto_instance *inst;
+
+	inst = aead_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
+	if (IS_ERR(inst))
+		goto out;
+
+	err = -EINVAL;
+	if (inst->alg.cra_aead.ivsize != inst->alg.cra_blocksize)
+		goto free_inst;
+
+	inst->alg.cra_aead.givencrypt = eseqiv_aead_givencrypt_first;
+
+	 inst->alg.cra_init = eseqiv_aead_init;
+	 inst->alg.cra_exit = aead_geniv_exit;
+
+	inst->alg.cra_ctxsize = inst->alg.cra_aead.ivsize;
+
+out:
+	return inst;
+
+free_inst:
+	aead_geniv_free(inst);
+	return ERR_PTR(err);
+}
+
+static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
+{
+	struct crypto_attr_type *algt;
+	struct crypto_instance *inst;
+	int err;
+
+	algt = crypto_get_attr_type(tb);
+	err = PTR_ERR(algt);
+	if (IS_ERR(algt))
+		return ERR_PTR(err);
+
+	err = crypto_get_default_rng();
+	if (err)
+		return ERR_PTR(err);
+
+	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+		inst = eseqiv_ablkcipher_alloc(tb);
+	else
+		inst = eseqiv_aead_alloc(tb);
+
+	if (IS_ERR(inst))
+		goto put_rng;
+
+	inst->alg.cra_ctxsize += sizeof(struct eseqiv_ctx);
+
+out:
+	return inst;
+
 put_rng:
 	crypto_put_default_rng();
 	goto out;

@@ -254,7 +471,10 @@ put_rng:
 
 static void eseqiv_free(struct crypto_instance *inst)
 {
-	skcipher_geniv_free(inst);
+	if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
+		skcipher_geniv_free(inst);
+	else
+		aead_geniv_free(inst);
 	crypto_put_default_rng();
 }
 

index 2548bf0..5e336b2 100644
@@ -82,6 +82,7 @@
 #define CRYPTO_TFM_REQ_WEAK_KEY		0x00000100
 #define CRYPTO_TFM_REQ_MAY_SLEEP	0x00000200
 #define CRYPTO_TFM_REQ_MAY_BACKLOG	0x00000400
+#define CRYPTO_TFM_REQ_SG_HAS_IV	0x00000800
 #define CRYPTO_TFM_RES_WEAK_KEY		0x00100000
 #define CRYPTO_TFM_RES_BAD_KEY_LEN   	0x00200000
 #define CRYPTO_TFM_RES_BAD_KEY_SCHED 	0x00400000

index 5793b64..aaa0fcc 100644
@@ -13,6 +13,7 @@
 #include <crypto/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/internal/aead.h>
 #include <crypto/authenc.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>

@@ -23,6 +24,10 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
+static char *wrapper = NULL;
+module_param(wrapper, charp, 0);
+MODULE_PARM_DESC(wrapper, "Wrapper template for AEAD algorithms");
+
 struct authenc_instance_ctx {
 	struct crypto_spawn auth;
 	struct crypto_skcipher_spawn enc;

@@ -157,15 +162,16 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
 
 	dstp = sg_page(dst);
 	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
+	cryptlen = req->cryptlen;
 
-	if (ivsize) {
+	if (ivsize && !(aead_request_flags(req) & CRYPTO_TFM_REQ_SG_HAS_IV)) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		authenc_chain(cipher, dst, vdst == iv + ivsize);
 		dst = cipher;
+		cryptlen += ivsize;
 	}
 
-	cryptlen = req->cryptlen + ivsize;
 	hash = crypto_authenc_hash(req, flags, dst, cryptlen);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);

@@ -369,6 +375,26 @@ static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm)
 	crypto_free_ablkcipher(ctx->enc);
 }
 
+static int crypto_authenc_set_wrapper(struct crypto_instance *inst)
+{
+	int err = 0;
+
+	if (!wrapper)
+		goto out;
+
+	if (!strcmp(wrapper, "pcrypt")) {
+		inst->alg.cra_type = &crypto_nivaead_type;
+		inst->alg.cra_aead.geniv = "eseqiv";
+		inst->alg.cra_aead.wrapper = "pcrypt";
+
+		goto out;
+	}
+
+	err = -EINVAL;
+out:
+	return err;
+}
+
 static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;

@@ -452,6 +478,9 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
 	inst->alg.cra_aead.decrypt = crypto_authenc_decrypt;
 	inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt;
 
+	if (crypto_authenc_set_wrapper(inst))
+		goto err_drop_enc;
+
 out:
 	crypto_mod_put(auth);
 	return inst;