crypto: reduce overhead when self-tests disabled
v2 cover crypto: reduce overhead when self-tests disabled
This patchset makes it so that the self-test code doesn't still slow things down when self-tests are disabled via the kconfig.
It also optimizes the registration of "internal" algorithms and silences a noisy log message.
Eric Biggers (6):
crypto: optimize algorithm registration when self-tests disabled
crypto: optimize registration of internal algorithms
crypto: compile out crypto_boot_test_finished when tests disabled
crypto: skip kdf_sp800108 self-test when tests disabled
crypto: silence noisy kdf_sp800108 self-test
crypto: compile out test-related algboss code when tests disabled
crypto/algapi.c | 160 ++++++++++++++++++++++++------------------ crypto/algboss.c | 26 +++---- crypto/api.c | 8 ++- crypto/internal.h | 13 +++- crypto/kdf_sp800108.c | 11 ++- 5 files changed, 124 insertions(+), 94 deletions(-)
base-commit: f67dd6ce0723ad013395f20a3f79d8a437d3f455
Signature · 1 line
2.38.1
PATCH v2 1/6 crypto: optimize algorithm registration when self-tests disabled
From: Eric Biggers <[email protected]>
Currently, registering an algorithm with the crypto API always causes a notification to be posted to the "cryptomgr", which then creates a kthread to self-test the algorithm. However, if self-tests are disabled in the kconfig (as is the default option), then this kthread just notifies waiters that the algorithm has been tested, then exits.
This causes a significant amount of overhead, especially in the kthread creation and destruction, which is not necessary at all. For example, in a quick test I found that booting a "minimum" x86_64 kernel with all the crypto options enabled (except for the self-tests) takes about 400ms until PID 1 can start. Of that, a full 13ms is spent just doing this pointless dance, involving a kthread being created, run, and destroyed over 200 times. That's over 3% of the entire kernel start time.
Fix this by just skipping the creation of the test larval and the posting of the registration notification entirely, when self-tests are disabled.
--- crypto/algapi.c | 151 ++++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 69 deletions(-)
crypto/algapi.c +82 -69
@@ -222,12 +222,62 @@ void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, }EXPORT_SYMBOL_GPL (crypto_remove_spawns); +static void crypto_alg_finish_registration (struct crypto_alg *alg, +bool fulfill_requests, +struct list_head *algs_to_put) +{ +struct crypto_alg *q; + +list_for_each_entry (q, &crypto_alg_list, cra_list) { +if (q == alg) +continue ; + +if (crypto_is_moribund (q)) +continue ; + +if (crypto_is_larval (q)) { +struct crypto_larval *larval = (void *)q; + +/* +* Check to see if either our generic name or +* specific name can satisfy the name requested +* by the larval entry q. +*/ +if (strcmp (alg->cra_name, q->cra_name) && +strcmp (alg->cra_driver_name, q->cra_name)) +continue ; + +if (larval->adult) +continue ; +if ((q->cra_flags ^ alg->cra_flags) & larval->mask) +continue ; + +if (fulfill_requests &&crypto_mod_get (alg)) + larval->adult = alg; +else + larval->adult =ERR_PTR (-EAGAIN ); + +continue ; + } + +if (strcmp (alg->cra_name, q->cra_name)) +continue ; + +if (strcmp (alg->cra_driver_name, q->cra_driver_name) && + q->cra_priority > alg->cra_priority) +continue ; + +crypto_remove_spawns (q, algs_to_put, alg); + } +} +static struct crypto_larval *crypto_alloc_test_larval (struct crypto_alg *alg) {struct crypto_larval *larval; -if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER )) -return NULL ; +if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER ) || +IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return NULL ;/* No self-test needed */ larval =crypto_larval_alloc (alg->cra_name, alg->cra_flags |CRYPTO_ALG_TESTED ,0 );
@@ -248,7 +298,8 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)return larval; } -static struct crypto_larval *__crypto_register_alg (struct crypto_alg *alg) +static struct crypto_larval * +__crypto_register_alg (struct crypto_alg *alg,struct list_head *algs_to_put) {struct crypto_alg *q;struct crypto_larval *larval;
@@ -259,9 +310,6 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)INIT_LIST_HEAD (&alg->cra_users); -/* No cheating! */ - alg->cra_flags &= ~CRYPTO_ALG_TESTED ; - ret = -EEXIST ;list_for_each_entry (q, &crypto_alg_list, cra_list) {
@@ -288,12 +336,17 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)list_add (&alg->cra_list, &crypto_alg_list); -if (larval) +crypto_stats_init (alg); + +if (larval) { +/* No cheating! */ + alg->cra_flags &= ~CRYPTO_ALG_TESTED ; +list_add (&larval->alg.cra_list, &crypto_alg_list); -else + }else { alg->cra_flags |=CRYPTO_ALG_TESTED ; - -crypto_stats_init (alg); +crypto_alg_finish_registration (alg, true, algs_to_put); + } out:return larval;
@@ -341,7 +394,10 @@ void crypto_alg_tested(const char *name, int err) alg->cra_flags |=CRYPTO_ALG_TESTED ; -/* Only satisfy larval waiters if we are the best. */ +/* +* If a higher-priority implementation of the same algorithm is +* currently being tested, then don't fulfill request larvals. +*/ best = true;list_for_each_entry (q, &crypto_alg_list, cra_list) {if (crypto_is_moribund (q) || !crypto_is_larval (q))
@@ -356,47 +412,7 @@ void crypto_alg_tested(const char *name, int err) } } -list_for_each_entry (q, &crypto_alg_list, cra_list) { -if (q == alg) -continue ; - -if (crypto_is_moribund (q)) -continue ; - -if (crypto_is_larval (q)) { -struct crypto_larval *larval = (void *)q; - -/* -* Check to see if either our generic name or -* specific name can satisfy the name requested -* by the larval entry q. -*/ -if (strcmp (alg->cra_name, q->cra_name) && -strcmp (alg->cra_driver_name, q->cra_name)) -continue ; - -if (larval->adult) -continue ; -if ((q->cra_flags ^ alg->cra_flags) & larval->mask) -continue ; - -if (best &&crypto_mod_get (alg)) - larval->adult = alg; -else - larval->adult =ERR_PTR (-EAGAIN ); - -continue ; - } - -if (strcmp (alg->cra_name, q->cra_name)) -continue ; - -if (strcmp (alg->cra_driver_name, q->cra_driver_name) && - q->cra_priority > alg->cra_priority) -continue ; - -crypto_remove_spawns (q, &list, alg); - } +crypto_alg_finish_registration (alg, best, &list); complete:complete_all (&test->completion);
@@ -423,7 +439,7 @@ EXPORT_SYMBOL_GPL(crypto_remove_final);int crypto_register_alg (struct crypto_alg *alg) {struct crypto_larval *larval; -bool test_started; +LIST_HEAD (algs_to_put);int err; alg->cra_flags &= ~CRYPTO_ALG_DEAD ;
@@ -432,17 +448,16 @@ int crypto_register_alg(struct crypto_alg *alg)return err;down_write (&crypto_alg_sem); - larval =__crypto_register_alg (alg); - test_started =static_key_enabled (&crypto_boot_test_finished); + larval =__crypto_register_alg (alg, &algs_to_put);if (!IS_ERR_OR_NULL (larval)) - larval->test_started = test_started; + larval->test_started =static_key_enabled (&crypto_boot_test_finished);up_write (&crypto_alg_sem); -if (IS_ERR_OR_NULL (larval)) +if (IS_ERR (larval))return PTR_ERR (larval); - -if (test_started) +if (larval && larval->test_started)crypto_wait_for_test (larval); +crypto_remove_final (&algs_to_put);return 0 ; }EXPORT_SYMBOL_GPL (crypto_register_alg);
@@ -619,6 +634,7 @@ int crypto_register_instance(struct crypto_template *tmpl,struct crypto_larval *larval;struct crypto_spawn *spawn;u32 fips_internal =0 ; +LIST_HEAD (algs_to_put);int err; err =crypto_check_alg (&inst->alg);
@@ -650,7 +666,7 @@ int crypto_register_instance(struct crypto_template *tmpl, inst->alg.cra_flags |= (fips_internal &CRYPTO_ALG_FIPS_INTERNAL ); - larval =__crypto_register_alg (&inst->alg); + larval =__crypto_register_alg (&inst->alg, &algs_to_put);if (IS_ERR (larval)) goto unlock;else if (larval)
@@ -662,15 +678,12 @@ int crypto_register_instance(struct crypto_template *tmpl, unlock:up_write (&crypto_alg_sem); - err =PTR_ERR (larval); -if (IS_ERR_OR_NULL (larval)) - goto err; - -crypto_wait_for_test (larval); - err =0 ; - -err: -return err; +if (IS_ERR (larval)) +return PTR_ERR (larval); +if (larval) +crypto_wait_for_test (larval); +crypto_remove_final (&algs_to_put); +return 0 ; }EXPORT_SYMBOL_GPL (crypto_register_instance);
base-commit: f67dd6ce0723ad013395f20a3f79d8a437d3f455
Signature · 1 line
2.38.1
Re: [PATCH v2 1/6] crypto: optimize algorithm registration when self-tests disabled
Quoted 16 lines · Eric Biggers
From: Eric Biggers <[email protected]>
Currently, registering an algorithm with the crypto API always causes a notification to be posted to the "cryptomgr", which then creates a kthread to self-test the algorithm. However, if self-tests are disabled in the kconfig (as is the default option), then this kthread just notifies waiters that the algorithm has been tested, then exits.
This causes a significant amount of overhead, especially in the kthread creation and destruction, which is not necessary at all. For example, in a quick test I found that booting a "minimum"
x86_64kernel with all the crypto options enabled (except for the self-tests) takes about 400ms until PID 1 can start. Of that, a full 13ms is spent just doing this pointless dance, involving a kthread being created, run, and destroyed over 200 times. That's over 3% of the entire kernel start time.
Quoted 8 lines
Fix this by just skipping the creation of the test larval and the posting of the registration notification entirely, when self-tests are disabled.
Signed-off-by Eric Biggers <[email protected]>--- crypto/algapi.c | 151 ++++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 69 deletions(-)
FYI, I realized that this patch breaks CRYPTO_MSG_ALG_LOADED (it isn't always sent now). So I'll have to send a new version at least for that.
- Eric
Herbert Xu
maintainerRe: [PATCH v2 1/6] crypto: optimize algorithm registration when self-tests disabled
In reply to Eric BiggersQuoted 13 lines · Eric Biggers
crypto/algapi.c +2
-3@@ -432,17 +448,16 @@ int crypto_register_alg(struct crypto_alg *alg)return err;down_write (&crypto_alg_sem); - larval =__crypto_register_alg (alg); - test_started =static_key_enabled (&crypto_boot_test_finished); + larval =__crypto_register_alg (alg, &algs_to_put);if (!IS_ERR_OR_NULL (larval)) - larval->test_started = test_started; + larval->test_started =static_key_enabled (&crypto_boot_test_finished);up_write (&crypto_alg_sem);
crypto/algapi.c int crypto_register_alg(struct crypto_alg *alg) · line 462
Quoted 10 lines
crypto/algapi.c +3
-3-if (IS_ERR_OR_NULL (larval)) +if (IS_ERR (larval))return PTR_ERR (larval); - -if (test_started) +if (larval && larval->test_started)crypto_wait_for_test (larval); +crypto_remove_final (&algs_to_put);return 0 ; }
Is it safe to test larval->test_started instead of the local test_started? This value could change from false to true behind your back.
Cheers,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
PATCH v2 2/6 crypto: optimize registration of internal algorithms
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Since algboss always skips testing of algorithms with the CRYPTO_ALG_INTERNAL flag, there is no need to go through the dance of creating the test kthread, which creates a lot of overhead. Instead, we can just directly finish the algorithm registration, like is now done when self-tests are disabled entirely.
--- crypto/algapi.c | 3 ++- crypto/algboss.c | 13 +------------ 2 files changed, 3 insertions(+), 13 deletions(-)
crypto/algapi.c +2 -1
@@ -276,7 +276,8 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)struct crypto_larval *larval;if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER ) || -IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS ) || + (alg->cra_flags &CRYPTO_ALG_INTERNAL ))return NULL ;/* No self-test needed */ larval = crypto_larval_alloc(alg->cra_name,
crypto/algboss.c +1 -12
@@ -181,12 +181,8 @@ static int cryptomgr_test(void *data) goto skiptest;#endif -if (type &CRYPTO_ALG_TESTED ) - goto skiptest; - err =alg_test (param->driver, param->alg, type,CRYPTO_ALG_TESTED ); -skiptest:crypto_alg_tested (param->driver, err);kfree (param);
@@ -197,7 +193,6 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) {struct task_struct *thread;struct crypto_test_param *param; -u32 type;if (!try_module_get (THIS_MODULE )) goto err;
@@ -208,13 +203,7 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg)memcpy (param->driver, alg->cra_driver_name,sizeof (param->driver));memcpy (param->alg, alg->cra_name,sizeof (param->alg)); - type = alg->cra_flags; - -/* Do not test internal algorithms. */ -if (type &CRYPTO_ALG_INTERNAL ) - type |=CRYPTO_ALG_TESTED ; - - param->type = type; + param->type = alg->cra_flags; thread =kthread_run (cryptomgr_test, param,"cryptomgr_test" );if (IS_ERR (thread))
Signature · 1 line
2.38.1
PATCH v2 3/6 crypto: compile out crypto_boot_test_finished when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
The crypto_boot_test_finished static key is unnecessary when self-tests are disabled in the kconfig, so optimize it out accordingly, along with the entirety of crypto_start_tests(). This mainly avoids the overhead of an unnecessary static_branch_enable() on every boot.
--- crypto/algapi.c | 10 ++++++++-- crypto/api.c | 8 +++++--- crypto/internal.h | 13 ++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-)
crypto/algapi.c +8 -2
@@ -451,7 +451,7 @@ int crypto_register_alg(struct crypto_alg *alg)down_write (&crypto_alg_sem); larval =__crypto_register_alg (alg, &algs_to_put);if (!IS_ERR_OR_NULL (larval)) - larval->test_started =static_key_enabled (&crypto_boot_test_finished); + larval->test_started =crypto_boot_test_finished ();up_write (&crypto_alg_sem);if (IS_ERR (larval))
@@ -1246,6 +1246,11 @@ void crypto_stats_skcipher_decrypt(unsigned int cryptlen, int ret,EXPORT_SYMBOL_GPL (crypto_stats_skcipher_decrypt);#endif +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +static void __init crypto_start_tests (void ) +{ +} +#else static void __init crypto_start_tests (void ) {for (;;) {
@@ -1281,8 +1286,9 @@ static void __init crypto_start_tests(void) crypto_wait_for_test(larval); } -static_branch_enable (&crypto_boot_test_finished); +static_branch_enable (&__crypto_boot_test_finished); } +#endif /* !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ static int __init crypto_algapi_init (void ) {
crypto/api.c +5 -3
@@ -31,8 +31,10 @@ EXPORT_SYMBOL_GPL(crypto_alg_sem);BLOCKING_NOTIFIER_HEAD (crypto_chain);EXPORT_SYMBOL_GPL (crypto_chain); -DEFINE_STATIC_KEY_FALSE (crypto_boot_test_finished); -EXPORT_SYMBOL_GPL (crypto_boot_test_finished); +#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +DEFINE_STATIC_KEY_FALSE (__crypto_boot_test_finished); +EXPORT_SYMBOL_GPL (__crypto_boot_test_finished); +#endif static struct crypto_alg *crypto_larval_wait (struct crypto_alg *alg);
@@ -205,7 +207,7 @@ static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg)struct crypto_larval *larval = (void *)alg;long timeout; -if (!static_branch_likely (&crypto_boot_test_finished)) +if (!crypto_boot_test_finished ())crypto_start_test (larval); timeout = wait_for_completion_killable_timeout(
crypto/internal.h +12 -1
@@ -47,7 +47,18 @@ extern struct list_head crypto_alg_list;extern struct rw_semaphore crypto_alg_sem;extern struct blocking_notifier_head crypto_chain; -DECLARE_STATIC_KEY_FALSE (crypto_boot_test_finished); +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +static inline bool crypto_boot_test_finished (void ) +{ +return true; +} +#else +DECLARE_STATIC_KEY_FALSE (__crypto_boot_test_finished); +static inline bool crypto_boot_test_finished (void ) +{ +return static_branch_likely (&__crypto_boot_test_finished); +} +#endif #ifdef CONFIG_PROC_FS void __init crypto_init_proc (void );
Signature · 1 line
2.38.1
PATCH v2 4/6 crypto: skip kdf_sp800108 self-test when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Make kdf_sp800108 honor the CONFIG_CRYPTO_MANAGER_DISABLE_TESTS kconfig option, so that it doesn't always waste time running its self-test.
--- crypto/kdf_sp800108.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
crypto/kdf_sp800108.c +6 -2
@@ -125,9 +125,13 @@ static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = {static int __init crypto_kdf108_init (void ) { -int ret =kdf_test (&kdf_ctr_hmac_sha256_tv_template[0 ],"hmac(sha256)" , - crypto_kdf108_setkey, crypto_kdf108_ctr_generate); +int ret; +if (IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return 0 ; + + ret =kdf_test (&kdf_ctr_hmac_sha256_tv_template[0 ],"hmac(sha256)" , + crypto_kdf108_setkey, crypto_kdf108_ctr_generate);if (ret) {if (fips_enabled) panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
Signature · 1 line
2.38.1
PATCH v2 5/6 crypto: silence noisy kdf_sp800108 self-test
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Make the kdf_sp800108 self-test only print a message on success when fips_enabled, so that it's consistent with testmgr.c and doesn't spam the kernel log with a message that isn't really important.
--- crypto/kdf_sp800108.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
crypto/kdf_sp800108.c +1 -1
@@ -140,7 +140,7 @@ static int __init crypto_kdf108_init(void)WARN (1 ,"alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n" , ret); - }else { + }else if (fips_enabled) {pr_info ("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n" ); }
Signature · 1 line
2.38.1
PATCH v2 6/6 crypto: compile out test-related algboss code when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
When CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is set, the code in algboss.c that handles CRYPTO_MSG_ALG_REGISTER is unnecessary, so make it be compiled out.
--- crypto/algboss.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
crypto/algboss.c +8 -5
@@ -171,15 +171,17 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval)return NOTIFY_OK ; } +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +static int cryptomgr_schedule_test (struct crypto_alg *alg) +{ +return NOTIFY_DONE ; +} +#else static int cryptomgr_test (void *data) {struct crypto_test_param *param = data;u32 type = param->type; -int err =0 ; - -#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS - goto skiptest; -#endif +int err; err =alg_test (param->driver, param->alg, type,CRYPTO_ALG_TESTED );
@@ -218,6 +220,7 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) err: return NOTIFY_OK; } +#endif /* !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ static int cryptomgr_notify (struct notifier_block *this,unsigned long msg,void *data)
Signature · 1 line
2.38.1
Herbert Xu
maintainerRe: [PATCH v2 6/6] crypto: compile out test-related algboss code when tests disabled
crypto/algboss.c static int cryptomgr_schedule_probe(struct crypto_larval *larval) · line 179
Quoted 10 lines · Eric Biggers
crypto/algboss.c +6
-0@@ -171,15 +171,17 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval)return NOTIFY_OK ; } +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +static int cryptomgr_schedule_test (struct crypto_alg *alg) +{ +return NOTIFY_DONE ; +} +#else
Could you please do this inline with an if statement rather than as #ifdefs? That is,
static int cryptomgr_schedule_test(struct crypto_alg *alg)
{
struct task_struct *thread;
struct crypto_test_param *param;
u32 type;
if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS))
return NOTIFY_DONE;Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt