crypto: introduce generic dynamic software fallback and EIP93 support

180 lines not on this page · raw message

255 lines not on this page · raw message

new file mode 100644
index 000000000000..09dc0fb38483
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _CRYPTO_BENCHMARK_H
+#define _CRYPTO_BENCHMARK_H
+
+#include <linux/types.h>
+
+struct aead_request;
+struct ahash_request;
+struct skcipher_request;
+
+/* Requests must use crypto_req_done() with struct crypto_wait callback data. */
+int crypto_benchmark_aead_cycles(struct aead_request *req, bool encrypt,
+				 unsigned int warmup_runs, unsigned int runs,
+				 u64 *total_cycles);
+int crypto_benchmark_ahash_cycles(struct ahash_request *req,
+				  unsigned int block_size,
+				  unsigned int update_size,
+				  unsigned int warmup_runs, unsigned int runs,
+				  u64 *total_cycles);
+int crypto_benchmark_skcipher_cycles(struct skcipher_request *req, bool encrypt,
+				     unsigned int warmup_runs,
+				     unsigned int runs, u64 *total_cycles);
+
+#endif /* _CRYPTO_BENCHMARK_H */

1,335 lines not on this page · raw message

96 lines not on this page · raw message

164 lines not on this page · raw message

new file mode 100644
index 000000000000..e3ba9a66abc7
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _EIP93_FALLBACK_H_
+#define _EIP93_FALLBACK_H_
+
+struct eip93_device;
+
+int eip93_fallback_register(struct eip93_device *eip93);
+void eip93_fallback_unregister(void);
+
+#endif /* _EIP93_FALLBACK_H_ */

index 1a8dabc4ada4..96ecffeb75df 100644
@@ -25,6 +25,7 @@
 #include "eip93-aes.h"
 #include "eip93-des.h"
 #include "eip93-aead.h"
+#include "eip93-fallback.h"
 #include "eip93-hash.h"
 
 static struct eip93_alg_template *eip93_algs[] = {

@@ -460,10 +461,12 @@ static int eip93_crypto_probe(struct platform_device *pdev)
 	eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH);
 
 	ret = eip93_register_algs(eip93, algo_flags);
-	if (ret) {
-		eip93_cleanup(eip93);
-		return ret;
-	}
+	if (ret)
+		goto err_cleanup;
+
+	ret = eip93_fallback_register(eip93);
+	if (ret)
+		goto err_unregister_algs;
 
 	ver = readl(eip93->base + EIP93_REG_PE_REVISION);
 	/* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */

@@ -476,6 +479,13 @@ static int eip93_crypto_probe(struct platform_device *pdev)
 		 readl(eip93->base + EIP93_REG_PE_OPTION_0));
 
 	return 0;
+
+err_unregister_algs:
+	eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs));
+err_cleanup:
+	eip93_cleanup(eip93);
+
+	return ret;
 }
 
 static void eip93_crypto_remove(struct platform_device *pdev)

@@ -485,6 +495,7 @@ static void eip93_crypto_remove(struct platform_device *pdev)
 
 	algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1);
 
+	eip93_fallback_unregister();
 	eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs));
 	eip93_cleanup(eip93);
 }

index af8830f1077c..40755d409e7a 100644
@@ -9,13 +9,16 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
+#include <linux/kref.h>
 #include <linux/limits.h>
+#include <linux/list.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/random.h>
 #include <linux/rtnetlink.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/sysfs.h>
 

@@ -48,10 +51,12 @@ struct crypto_fallback {
 	struct attribute_group sysfs_group;
 	struct attribute **sysfs_attrs;
 	struct device_attribute enabled_attr;
+	struct kref refcount;
 	struct mutex lock; /* Serializes tuning and sysfs writes. */
-	struct fallback_alg *algs;
+	spinlock_t alg_lock; /* Protects algs and fallback_alg state. */
+	struct list_head algs;
+	struct module *owner;
 	unsigned int num_groups;
-	unsigned int num_algs;
 	bool enabled;
 };
 

@@ -422,6 +427,9 @@ static int crypto_fallback_tune_group(const struct crypto_fallback_group *group,
 	return 0;
 }
 
+static int fallback_register_algs(struct crypto_fallback *fallback);
+static void fallback_unregister_algs(struct crypto_fallback *fallback);
+
 static ssize_t fallback_enabled_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
 {

@@ -451,9 +459,6 @@ static ssize_t fallback_enabled_store(struct device *dev,
 		goto out_unlock;
 
 	if (enabled) {
-		for (i = 0; i < fallback->num_groups; i++)
-			WRITE_ONCE(fallback->groups[i].threshold.value, 0);
-
 		for (i = 0; i < fallback->num_groups; i++) {
 			struct crypto_fallback_group_state *state;
 			int threshold;

@@ -463,9 +468,10 @@ static ssize_t fallback_enabled_store(struct device *dev,
 							 &threshold);
 			if (err) {
 				dev_warn(fallback->dev,
-					"%s benchmark failed: %d; using hardware\n",
-					state->group.name, err);
+					 "%s benchmark failed: %d; using hardware\n",
+					 state->group.name, err);
 				threshold = 0;
+				err = 0;
 			}
 			WRITE_ONCE(state->value, threshold);
 			dev_info(fallback->dev,

@@ -476,28 +482,39 @@ static ssize_t fallback_enabled_store(struct device *dev,
 		for (i = 0; i < fallback->num_groups; i++)
 			WRITE_ONCE(fallback->groups[i].threshold.value,
 				   fallback->groups[i].value);
+
+		if (!fallback->enabled) {
+			err = fallback_register_algs(fallback);
+			if (err) {
+				for (i = 0; i < fallback->num_groups; i++)
+					WRITE_ONCE(fallback->groups[i].threshold.value,
+						   0);
+				goto out_unlock;
+			}
+		}
 	} else {
+		WRITE_ONCE(fallback->enabled, false);
 		for (i = 0; i < fallback->num_groups; i++)
 			WRITE_ONCE(fallback->groups[i].threshold.value, 0);
+		fallback_unregister_algs(fallback);
 	}
 	WRITE_ONCE(fallback->enabled, enabled);
 
 out_unlock:
 	mutex_unlock(&fallback->lock);
 
-	return count;
+	return err ?: count;
 }
 
-static void crypto_fallback_free(struct crypto_fallback *fallback)
+static void crypto_fallback_free(struct kref *ref)
 {
+	struct crypto_fallback *fallback;
 	unsigned int i;
 
-	if (!fallback)
-		return;
+	fallback = container_of(ref, struct crypto_fallback, refcount);
 
 	for (i = 0; i < fallback->num_groups; i++)
 		kfree_const(fallback->groups[i].group.name);
-	kfree(fallback->algs);
 	kfree(fallback->sysfs_attrs);
 	kfree(fallback->groups);
 	mutex_destroy(&fallback->lock);

@@ -506,7 +523,8 @@ static void crypto_fallback_free(struct crypto_fallback *fallback)
 }
 
 static struct crypto_fallback *
-fallback_alloc(struct device *dev, const struct crypto_fallback_group *groups,
+fallback_alloc(struct module *owner, struct device *dev,
+	       const struct crypto_fallback_group *groups,
 	       unsigned int num_groups)
 {
 	struct crypto_fallback *fallback;

@@ -520,8 +538,12 @@ fallback_alloc(struct device *dev, const struct crypto_fallback_group *groups,
 	if (!fallback)
 		return ERR_PTR(-ENOMEM);
 
+	kref_init(&fallback->refcount);
 	mutex_init(&fallback->lock);
+	spin_lock_init(&fallback->alg_lock);
+	INIT_LIST_HEAD(&fallback->algs);
 	fallback->dev = get_device(dev);
+	fallback->owner = owner;
 	fallback->num_groups = num_groups;
 	fallback->groups =
 		kcalloc(num_groups, sizeof(*fallback->groups), GFP_KERNEL);

@@ -583,20 +605,11 @@ fallback_alloc(struct device *dev, const struct crypto_fallback_group *groups,
 	return fallback;
 
 err_free:
-	crypto_fallback_free(fallback);
+	kref_put(&fallback->refcount, crypto_fallback_free);
 
 	return ERR_PTR(err);
 }
 
-static void fallback_release(struct crypto_fallback *fallback)
-{
-	if (!fallback || IS_ERR(fallback))
-		return;
-
-	sysfs_remove_group(&fallback->dev->kobj, &fallback->sysfs_group);
-	crypto_fallback_free(fallback);
-}
-
 static bool fallback_use_software(const struct fallback_threshold *threshold,
 				  unsigned int input_size)
 {

@@ -630,10 +643,20 @@ struct fallback_ahash_state {
 	u8 state[] CRYPTO_MINALIGN_ATTR;
 };
 
+/*
+ * Allocate proxy algorithms per enable cycle because their destruction can be
+ * deferred until transforms allocated before disable release them.
+ */
 struct fallback_alg {
+	struct list_head list;
+	struct crypto_fallback *fallback;
+	struct kref refcount;
 	const struct fallback_threshold *threshold;
 	char driver_name[CRYPTO_MAX_ALG_NAME];
 	enum crypto_fallback_alg_type type;
+	bool registration_complete;
+	bool registered;
+	bool dead;
 	union {
 		struct skcipher_alg skcipher;
 		struct aead_alg aead;

@@ -641,6 +664,87 @@ struct fallback_alg {
 	} alg;
 };
 
+static void fallback_alg_release(struct kref *ref)
+{
+	struct fallback_alg *alg;
+	struct crypto_fallback *fallback;
+
+	alg = container_of(ref, struct fallback_alg, refcount);
+	fallback = alg->fallback;
+	kfree(alg);
+	kref_put(&fallback->refcount, crypto_fallback_free);
+}
+
+static void fallback_alg_destroy(struct fallback_alg *alg)
+{
+	struct crypto_fallback *fallback = alg->fallback;
+	unsigned long flags;
+	bool release = false;
+
+	spin_lock_irqsave(&fallback->alg_lock, flags);
+	alg->dead = true;
+	alg->registered = false;
+	if (alg->registration_complete) {
+		list_del_init(&alg->list);
+		release = true;
+	}
+	spin_unlock_irqrestore(&fallback->alg_lock, flags);
+
+	if (release)
+		kref_put(&alg->refcount, fallback_alg_release);
+}
+
+static void fallback_skcipher_destroy(struct crypto_alg *base)
+{
+	struct skcipher_alg *alg;
+
+	alg = container_of(base, struct skcipher_alg, base);
+	fallback_alg_destroy(container_of(alg, struct fallback_alg,
+					  alg.skcipher));
+}
+
+static void fallback_aead_destroy(struct crypto_alg *base)
+{
+	struct aead_alg *alg;
+
+	alg = container_of(base, struct aead_alg, base);
+	fallback_alg_destroy(container_of(alg, struct fallback_alg, alg.aead));
+}
+
+static void fallback_ahash_destroy(struct crypto_alg *base)
+{
+	struct hash_alg_common *halg;
+	struct ahash_alg *alg;
+
+	halg = container_of(base, struct hash_alg_common, base);
+	alg = container_of(halg, struct ahash_alg, halg);
+	fallback_alg_destroy(container_of(alg, struct fallback_alg, alg.ahash));
+}
+
+static int fallback_alg_complete_registration(struct fallback_alg *alg, int err)
+{
+	struct crypto_fallback *fallback = alg->fallback;
+	unsigned long flags;
+	bool release = false;
+
+	spin_lock_irqsave(&fallback->alg_lock, flags);
+	alg->registration_complete = true;
+	if (err || alg->dead) {
+		list_del_init(&alg->list);
+		release = true;
+		if (!err)
+			err = -ENODEV;
+	} else {
+		alg->registered = true;
+	}
+	spin_unlock_irqrestore(&fallback->alg_lock, flags);
+
+	if (release)
+		kref_put(&alg->refcount, fallback_alg_release);
+
+	return err;
+}
+
 static struct fallback_alg *fallback_skcipher_alg(struct crypto_skcipher *tfm)
 {
 	return container_of(crypto_skcipher_alg(tfm), struct fallback_alg,

@@ -1068,6 +1172,7 @@ static int fallback_register_skcipher(struct fallback_alg *entry,
 	if (err)
 		goto out_free_software;
 	fallback_init_base(&alg->base, &hcommon->base, &scommon->base, owner);
+	alg->base.cra_destroy = fallback_skcipher_destroy;
 	alg->min_keysize = max(hcommon->min_keysize, scommon->min_keysize);
 	alg->max_keysize = min(hcommon->max_keysize, scommon->max_keysize);
 	if (alg->min_keysize > alg->max_keysize) {

@@ -1129,6 +1234,7 @@ static int fallback_register_aead(struct fallback_alg *entry,
 	if (err)
 		goto out_free_software;
 	fallback_init_base(&alg->base, &halg->base, &salg->base, owner);
+	alg->base.cra_destroy = fallback_aead_destroy;
 	alg->ivsize = halg->ivsize;
 	alg->maxauthsize = min(halg->maxauthsize, salg->maxauthsize);
 	alg->chunksize = max(halg->chunksize, salg->chunksize);

@@ -1187,6 +1293,7 @@ static int fallback_register_ahash(struct fallback_alg *entry,
 		goto out_free_software;
 	fallback_init_base(&alg->halg.base, &hcommon->base, &scommon->base,
 			   owner);
+	alg->halg.base.cra_destroy = fallback_ahash_destroy;
 	alg->halg.base.cra_flags |= CRYPTO_ALG_REQ_VIRT;
 	alg->halg.base.cra_flags |= hcommon->base.cra_flags &
 				    CRYPTO_ALG_OPTIONAL_KEY;

@@ -1246,87 +1353,124 @@ static void fallback_unregister_alg(struct fallback_alg *alg)
 	}
 }
 
-struct crypto_fallback *
-crypto_fallback_register(struct module *owner, struct device *dev,
-			 const struct crypto_fallback_group *groups,
-			 unsigned int num_groups)
+static void fallback_unregister_algs(struct crypto_fallback *fallback)
 {
-	struct crypto_fallback *fallback;
-	unsigned int num_algs = 0;
-	unsigned int i, j;
-	int err;
+	struct fallback_alg *alg;
+	struct fallback_alg *candidate;
+	unsigned long flags;
+
+	for (;;) {
+		alg = NULL;
+		spin_lock_irqsave(&fallback->alg_lock, flags);
+		list_for_each_entry(candidate, &fallback->algs, list) {
+			if (!candidate->registered)
+				continue;
 
-	if (!dev || !groups || !num_groups)
-		return ERR_PTR(-EINVAL);
+			alg = candidate;
+			alg->registered = false;
+			kref_get(&alg->refcount);
+			break;
+		}
+		spin_unlock_irqrestore(&fallback->alg_lock, flags);
 
-	for (i = 0; i < num_groups; i++) {
-		if (!groups[i].algs || !groups[i].num_algs)
-			return ERR_PTR(-EINVAL);
-		if (groups[i].num_algs > UINT_MAX - num_algs)
-			return ERR_PTR(-EOVERFLOW);
-		num_algs += groups[i].num_algs;
+		if (!alg)
+			break;
+
+		fallback_unregister_alg(alg);
+		kref_put(&alg->refcount, fallback_alg_release);
 	}
+}
 
-	fallback = fallback_alloc(dev, groups, num_groups);
-	if (IS_ERR(fallback))
-		return fallback;
+static int fallback_register_algs(struct crypto_fallback *fallback)
+{
+	unsigned int i, j;
+	int err;
 
-	fallback->algs = kcalloc(num_algs, sizeof(*fallback->algs), GFP_KERNEL);
-	if (!fallback->algs) {
-		err = -ENOMEM;
-		goto err_release;
-	}
-	for (i = 0; i < num_groups; i++) {
-		const struct fallback_threshold *threshold;
+	for (i = 0; i < fallback->num_groups; i++) {
+		const struct crypto_fallback_group *group;
 
-		threshold = &fallback->groups[i].threshold;
-		for (j = 0; j < groups[i].num_algs; j++) {
-			const char *driver_name = groups[i].algs[j];
-			struct fallback_alg *alg =
-				&fallback->algs[fallback->num_algs];
+		group = &fallback->groups[i].group;
+		for (j = 0; j < group->num_algs; j++) {
+			struct fallback_alg *alg;
+			unsigned long flags;
 
-			if (!driver_name) {
-				err = -EINVAL;
+			alg = kzalloc_obj(*alg, GFP_KERNEL);
+			if (!alg) {
+				err = -ENOMEM;
 				goto err_unregister_algs;
 			}
 
-			alg->type = groups[i].benchmark.type;
-			alg->threshold = threshold;
-			if (strscpy(alg->driver_name, driver_name,
+			INIT_LIST_HEAD(&alg->list);
+			kref_init(&alg->refcount);
+			alg->fallback = fallback;
+			alg->threshold = &fallback->groups[i].threshold;
+			alg->type = group->benchmark.type;
+			if (strscpy(alg->driver_name, group->algs[j],
 				    sizeof(alg->driver_name)) < 0) {
+				kfree(alg);
 				err = -ENAMETOOLONG;
 				goto err_unregister_algs;
 			}
 
-			err = fallback_register_alg(alg, owner);
+			kref_get(&fallback->refcount);
+			spin_lock_irqsave(&fallback->alg_lock, flags);
+			list_add_tail(&alg->list, &fallback->algs);
+			spin_unlock_irqrestore(&fallback->alg_lock, flags);
+
+			err = fallback_register_alg(alg, fallback->owner);
+			err = fallback_alg_complete_registration(alg, err);
 			if (err == -ENOENT)
 				continue;
 			if (err)
 				goto err_unregister_algs;
-			fallback->num_algs++;
 		}
 	}
 
-	return fallback;
+	return 0;
 
 err_unregister_algs:
-	while (fallback->num_algs)
-		fallback_unregister_alg(&fallback->algs[--fallback->num_algs]);
-err_release:
-	fallback_release(fallback);
+	fallback_unregister_algs(fallback);
 
-	return ERR_PTR(err);
+	return err;
+}
+
+struct crypto_fallback *
+crypto_fallback_register(struct module *owner, struct device *dev,
+			 const struct crypto_fallback_group *groups,
+			 unsigned int num_groups)
+{
+	unsigned int i, j;
+
+	if (!owner || !dev || !groups || !num_groups)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < num_groups; i++) {
+		if (!groups[i].algs || !groups[i].num_algs)
+			return ERR_PTR(-EINVAL);
+		for (j = 0; j < groups[i].num_algs; j++)
+			if (!groups[i].algs[j])
+				return ERR_PTR(-EINVAL);
+	}
+
+	return fallback_alloc(owner, dev, groups, num_groups);
 }
 EXPORT_SYMBOL_GPL(crypto_fallback_register);
 
 void crypto_fallback_unregister(struct crypto_fallback *fallback)
 {
+	unsigned int i;
+
 	if (!fallback || IS_ERR(fallback))
 		return;
 
-	while (fallback->num_algs)
-		fallback_unregister_alg(&fallback->algs[--fallback->num_algs]);
-	fallback_release(fallback);
+	sysfs_remove_group(&fallback->dev->kobj, &fallback->sysfs_group);
+	mutex_lock(&fallback->lock);
+	WRITE_ONCE(fallback->enabled, false);
+	for (i = 0; i < fallback->num_groups; i++)
+		WRITE_ONCE(fallback->groups[i].threshold.value, 0);
+	fallback_unregister_algs(fallback);
+	mutex_unlock(&fallback->lock);
+	kref_put(&fallback->refcount, crypto_fallback_free);
 }
 EXPORT_SYMBOL_GPL(crypto_fallback_unregister);