crypto: reduce overhead when self-tests disabled
v3 cover crypto: reduce overhead when self-tests disabled
This patchset makes it so that the self-test code doesn't still slow things down when self-tests are disabled via the kconfig.
It also optimizes the registration of "internal" algorithms and silences a noisy log message.
Changed in v3:
- Made sure CRYPTO_MSG_ALG_LOADED still always gets sent.
- Fixed a race condition with larval->test_started.
- Used IS_ENABLED() in a couple places to avoid #ifdefs.
Eric Biggers (6):
crypto: optimize algorithm registration when self-tests disabled
crypto: optimize registration of internal algorithms
crypto: compile out crypto_boot_test_finished when tests disabled
crypto: skip kdf_sp800108 self-test when tests disabled
crypto: silence noisy kdf_sp800108 self-test
crypto: compile out test-related algboss code when tests disabled
crypto/algapi.c | 160 ++++++++++++++++++++++++------------------ crypto/algboss.c | 22 ++---- crypto/api.c | 11 ++- crypto/internal.h | 20 +++++- crypto/kdf_sp800108.c | 10 ++- 5 files changed, 127 insertions(+), 96 deletions(-)
base-commit: 557ffd5a4726f8b6f0dd1d4b632ae02c1c063233
Signature · 1 line
2.38.1
PATCH v3 1/6 crypto: optimize algorithm registration when self-tests disabled
From: Eric Biggers <[email protected]>
Currently, registering an algorithm with the crypto API always causes a notification to be posted to the "cryptomgr", which then creates a kthread to self-test the algorithm. However, if self-tests are disabled in the kconfig (as is the default option), then this kthread just notifies waiters that the algorithm has been tested, then exits.
This causes a significant amount of overhead, especially in the kthread creation and destruction, which is not necessary at all. For example, in a quick test I found that booting a "minimum" x86_64 kernel with all the crypto options enabled (except for the self-tests) takes about 400ms until PID 1 can start. Of that, a full 13ms is spent just doing this pointless dance, involving a kthread being created, run, and destroyed over 200 times. That's over 3% of the entire kernel start time.
Fix this by just skipping the creation of the test larval and the posting of the registration notification entirely, when self-tests are disabled.
--- crypto/algapi.c | 154 +++++++++++++++++++++++++++--------------------- crypto/api.c | 3 - 2 files changed, 86 insertions(+), 71 deletions(-)
crypto/algapi.c +86 -68
@@ -222,12 +222,64 @@ void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list, }EXPORT_SYMBOL_GPL (crypto_remove_spawns); +static void crypto_alg_finish_registration (struct crypto_alg *alg, +bool fulfill_requests, +struct list_head *algs_to_put) +{ +struct crypto_alg *q; + +list_for_each_entry (q, &crypto_alg_list, cra_list) { +if (q == alg) +continue ; + +if (crypto_is_moribund (q)) +continue ; + +if (crypto_is_larval (q)) { +struct crypto_larval *larval = (void *)q; + +/* +* Check to see if either our generic name or +* specific name can satisfy the name requested +* by the larval entry q. +*/ +if (strcmp (alg->cra_name, q->cra_name) && +strcmp (alg->cra_driver_name, q->cra_name)) +continue ; + +if (larval->adult) +continue ; +if ((q->cra_flags ^ alg->cra_flags) & larval->mask) +continue ; + +if (fulfill_requests &&crypto_mod_get (alg)) + larval->adult = alg; +else + larval->adult =ERR_PTR (-EAGAIN ); + +continue ; + } + +if (strcmp (alg->cra_name, q->cra_name)) +continue ; + +if (strcmp (alg->cra_driver_name, q->cra_driver_name) && + q->cra_priority > alg->cra_priority) +continue ; + +crypto_remove_spawns (q, algs_to_put, alg); + } + +crypto_notify (CRYPTO_MSG_ALG_LOADED , alg); +} +static struct crypto_larval *crypto_alloc_test_larval (struct crypto_alg *alg) {struct crypto_larval *larval; -if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER )) -return NULL ; +if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER ) || +IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return NULL ;/* No self-test needed */ larval =crypto_larval_alloc (alg->cra_name, alg->cra_flags |CRYPTO_ALG_TESTED ,0 );
@@ -248,7 +300,8 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)return larval; } -static struct crypto_larval *__crypto_register_alg (struct crypto_alg *alg) +static struct crypto_larval * +__crypto_register_alg (struct crypto_alg *alg,struct list_head *algs_to_put) {struct crypto_alg *q;struct crypto_larval *larval;
@@ -259,9 +312,6 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)INIT_LIST_HEAD (&alg->cra_users); -/* No cheating! */ - alg->cra_flags &= ~CRYPTO_ALG_TESTED ; - ret = -EEXIST ;list_for_each_entry (q, &crypto_alg_list, cra_list) {
@@ -288,12 +338,17 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)list_add (&alg->cra_list, &crypto_alg_list); -if (larval) +crypto_stats_init (alg); + +if (larval) { +/* No cheating! */ + alg->cra_flags &= ~CRYPTO_ALG_TESTED ; +list_add (&larval->alg.cra_list, &crypto_alg_list); -else + }else { alg->cra_flags |=CRYPTO_ALG_TESTED ; - -crypto_stats_init (alg); +crypto_alg_finish_registration (alg, true, algs_to_put); + } out:return larval;
@@ -341,7 +396,10 @@ void crypto_alg_tested(const char *name, int err) alg->cra_flags |=CRYPTO_ALG_TESTED ; -/* Only satisfy larval waiters if we are the best. */ +/* +* If a higher-priority implementation of the same algorithm is +* currently being tested, then don't fulfill request larvals. +*/ best = true;list_for_each_entry (q, &crypto_alg_list, cra_list) {if (crypto_is_moribund (q) || !crypto_is_larval (q))
@@ -356,47 +414,7 @@ void crypto_alg_tested(const char *name, int err) } } -list_for_each_entry (q, &crypto_alg_list, cra_list) { -if (q == alg) -continue ; - -if (crypto_is_moribund (q)) -continue ; - -if (crypto_is_larval (q)) { -struct crypto_larval *larval = (void *)q; - -/* -* Check to see if either our generic name or -* specific name can satisfy the name requested -* by the larval entry q. -*/ -if (strcmp (alg->cra_name, q->cra_name) && -strcmp (alg->cra_driver_name, q->cra_name)) -continue ; - -if (larval->adult) -continue ; -if ((q->cra_flags ^ alg->cra_flags) & larval->mask) -continue ; - -if (best &&crypto_mod_get (alg)) - larval->adult = alg; -else - larval->adult =ERR_PTR (-EAGAIN ); - -continue ; - } - -if (strcmp (alg->cra_name, q->cra_name)) -continue ; - -if (strcmp (alg->cra_driver_name, q->cra_driver_name) && - q->cra_priority > alg->cra_priority) -continue ; - -crypto_remove_spawns (q, &list, alg); - } +crypto_alg_finish_registration (alg, best, &list); complete:complete_all (&test->completion);
@@ -423,7 +441,8 @@ EXPORT_SYMBOL_GPL(crypto_remove_final);int crypto_register_alg (struct crypto_alg *alg) {struct crypto_larval *larval; -bool test_started; +LIST_HEAD (algs_to_put); +bool test_started = false;int err; alg->cra_flags &= ~CRYPTO_ALG_DEAD ;
@@ -432,17 +451,18 @@ int crypto_register_alg(struct crypto_alg *alg)return err;down_write (&crypto_alg_sem); - larval =__crypto_register_alg (alg); - test_started =static_key_enabled (&crypto_boot_test_finished); -if (!IS_ERR_OR_NULL (larval)) + larval =__crypto_register_alg (alg, &algs_to_put); +if (!IS_ERR_OR_NULL (larval)) { + test_started =static_key_enabled (&crypto_boot_test_finished); larval->test_started = test_started; + }up_write (&crypto_alg_sem); -if (IS_ERR_OR_NULL (larval)) +if (IS_ERR (larval))return PTR_ERR (larval); -if (test_started)crypto_wait_for_test (larval); +crypto_remove_final (&algs_to_put);return 0 ; }EXPORT_SYMBOL_GPL (crypto_register_alg);
@@ -619,6 +639,7 @@ int crypto_register_instance(struct crypto_template *tmpl,struct crypto_larval *larval;struct crypto_spawn *spawn;u32 fips_internal =0 ; +LIST_HEAD (algs_to_put);int err; err =crypto_check_alg (&inst->alg);
@@ -650,7 +671,7 @@ int crypto_register_instance(struct crypto_template *tmpl, inst->alg.cra_flags |= (fips_internal &CRYPTO_ALG_FIPS_INTERNAL ); - larval =__crypto_register_alg (&inst->alg); + larval =__crypto_register_alg (&inst->alg, &algs_to_put);if (IS_ERR (larval)) goto unlock;else if (larval)
@@ -662,15 +683,12 @@ int crypto_register_instance(struct crypto_template *tmpl, unlock:up_write (&crypto_alg_sem); - err =PTR_ERR (larval); -if (IS_ERR_OR_NULL (larval)) - goto err; - -crypto_wait_for_test (larval); - err =0 ; - -err: -return err; +if (IS_ERR (larval)) +return PTR_ERR (larval); +if (larval) +crypto_wait_for_test (larval); +crypto_remove_final (&algs_to_put); +return 0 ; }EXPORT_SYMBOL_GPL (crypto_register_instance);
crypto/api.c +0 -3
@@ -172,9 +172,6 @@ void crypto_wait_for_test(struct crypto_larval *larval) err =wait_for_completion_killable (&larval->completion);WARN_ON (err); -if (!err) -crypto_notify (CRYPTO_MSG_ALG_LOADED , larval); - out:crypto_larval_kill (&larval->alg); }
base-commit: 557ffd5a4726f8b6f0dd1d4b632ae02c1c063233
Signature · 1 line
2.38.1
PATCH v3 2/6 crypto: optimize registration of internal algorithms
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Since algboss always skips testing of algorithms with the CRYPTO_ALG_INTERNAL flag, there is no need to go through the dance of creating the test kthread, which creates a lot of overhead. Instead, we can just directly finish the algorithm registration, like is now done when self-tests are disabled entirely.
--- crypto/algapi.c | 3 ++- crypto/algboss.c | 13 +------------ 2 files changed, 3 insertions(+), 13 deletions(-)
crypto/algapi.c +2 -1
@@ -278,7 +278,8 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)struct crypto_larval *larval;if (!IS_ENABLED (CONFIG_CRYPTO_MANAGER ) || -IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS ) || + (alg->cra_flags &CRYPTO_ALG_INTERNAL ))return NULL ;/* No self-test needed */ larval = crypto_larval_alloc(alg->cra_name,
crypto/algboss.c +1 -12
@@ -181,12 +181,8 @@ static int cryptomgr_test(void *data) goto skiptest;#endif -if (type &CRYPTO_ALG_TESTED ) - goto skiptest; - err =alg_test (param->driver, param->alg, type,CRYPTO_ALG_TESTED ); -skiptest:crypto_alg_tested (param->driver, err);kfree (param);
@@ -197,7 +193,6 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) {struct task_struct *thread;struct crypto_test_param *param; -u32 type;if (!try_module_get (THIS_MODULE )) goto err;
@@ -208,13 +203,7 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg)memcpy (param->driver, alg->cra_driver_name,sizeof (param->driver));memcpy (param->alg, alg->cra_name,sizeof (param->alg)); - type = alg->cra_flags; - -/* Do not test internal algorithms. */ -if (type &CRYPTO_ALG_INTERNAL ) - type |=CRYPTO_ALG_TESTED ; - - param->type = type; + param->type = alg->cra_flags; thread =kthread_run (cryptomgr_test, param,"cryptomgr_test" );if (IS_ERR (thread))
Signature · 1 line
2.38.1
Re: [PATCH v3 2/6] crypto: optimize registration of internal algorithms
Quoted 5 lines · Eric Biggers
crypto/algboss.c +0
-0
crypto/algboss.c static int cryptomgr_test(void *data) · line 186
Quoted 10 lines
crypto/algboss.c +0
-4@@ -181,12 +181,8 @@ static int cryptomgr_test(void *data) goto skiptest;#endif -if (type &CRYPTO_ALG_TESTED ) - goto skiptest; - err =alg_test (param->driver, param->alg, type,CRYPTO_ALG_TESTED ); -skiptest:crypto_alg_tested (param->driver, err);
Ard pointed out that there's a bisection hazard here, since this patch deletes the skiptest label, but the last goto to it isn't deleted until patch 6. Sorry about that. Herbert, do you want to fix this by rebasing, or is it too late?
- Eric
Herbert Xu
maintainerRe: [PATCH v3 2/6] crypto: optimize registration of internal algorithms
On Wed, Nov 30, 2022 at 07:04:23PM +0000, Eric Biggers wrote:
.
Quoted 3 lines
Ard pointed out that there's a bisection hazard here, since this patch deletes the skiptest label, but the last goto to it isn't deleted until patch 6. Sorry about that. Herbert, do you want to fix this by rebasing, or is it too late?
I don't think it's worth worrying about.
Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
PATCH v3 3/6 crypto: compile out crypto_boot_test_finished when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
The crypto_boot_test_finished static key is unnecessary when self-tests are disabled in the kconfig, so optimize it out accordingly, along with the entirety of crypto_start_tests(). This mainly avoids the overhead of an unnecessary static_branch_enable() on every boot.
--- crypto/algapi.c | 7 +++++-- crypto/api.c | 8 +++++--- crypto/internal.h | 20 +++++++++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-)
crypto/algapi.c +5 -2
@@ -454,7 +454,7 @@ int crypto_register_alg(struct crypto_alg *alg)down_write (&crypto_alg_sem); larval =__crypto_register_alg (alg, &algs_to_put);if (!IS_ERR_OR_NULL (larval)) { - test_started =static_key_enabled (&crypto_boot_test_finished); + test_started =crypto_boot_test_finished (); larval->test_started = test_started; }up_write (&crypto_alg_sem);
@@ -1253,6 +1253,9 @@ EXPORT_SYMBOL_GPL(crypto_stats_skcipher_decrypt);static void __init crypto_start_tests (void ) { +if (IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return ; +for (;;) {struct crypto_larval *larval =NULL ;struct crypto_alg *q;
@@ -1286,7 +1289,7 @@ static void __init crypto_start_tests(void)crypto_wait_for_test (larval); } -static_branch_enable (&crypto_boot_test_finished); +set_crypto_boot_test_finished (); }static int __init crypto_algapi_init (void )
crypto/api.c +5 -3
@@ -31,8 +31,10 @@ EXPORT_SYMBOL_GPL(crypto_alg_sem);BLOCKING_NOTIFIER_HEAD (crypto_chain);EXPORT_SYMBOL_GPL (crypto_chain); -DEFINE_STATIC_KEY_FALSE (crypto_boot_test_finished); -EXPORT_SYMBOL_GPL (crypto_boot_test_finished); +#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +DEFINE_STATIC_KEY_FALSE (__crypto_boot_test_finished); +EXPORT_SYMBOL_GPL (__crypto_boot_test_finished); +#endif static struct crypto_alg *crypto_larval_wait (struct crypto_alg *alg);
@@ -202,7 +204,7 @@ static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg)struct crypto_larval *larval = (void *)alg;long timeout; -if (!static_branch_likely (&crypto_boot_test_finished)) +if (!crypto_boot_test_finished ())crypto_start_test (larval); timeout = wait_for_completion_killable_timeout(
crypto/internal.h +19 -1
@@ -47,7 +47,25 @@ extern struct list_head crypto_alg_list;extern struct rw_semaphore crypto_alg_sem;extern struct blocking_notifier_head crypto_chain; -DECLARE_STATIC_KEY_FALSE (crypto_boot_test_finished); +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +static inline bool crypto_boot_test_finished (void ) +{ +return true; +} +static inline void set_crypto_boot_test_finished (void ) +{ +} +#else +DECLARE_STATIC_KEY_FALSE (__crypto_boot_test_finished); +static inline bool crypto_boot_test_finished (void ) +{ +return static_branch_likely (&__crypto_boot_test_finished); +} +static inline void set_crypto_boot_test_finished (void ) +{ +static_branch_enable (&__crypto_boot_test_finished); +} +#endif /* !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ #ifdef CONFIG_PROC_FS void __init crypto_init_proc (void );
Signature · 1 line
2.38.1
PATCH v3 4/6 crypto: skip kdf_sp800108 self-test when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Make kdf_sp800108 honor the CONFIG_CRYPTO_MANAGER_DISABLE_TESTS kconfig option, so that it doesn't always waste time running its self-test.
--- crypto/kdf_sp800108.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
crypto/kdf_sp800108.c +6 -2
@@ -125,9 +125,13 @@ static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = {static int __init crypto_kdf108_init (void ) { -int ret =kdf_test (&kdf_ctr_hmac_sha256_tv_template[0 ],"hmac(sha256)" , - crypto_kdf108_setkey, crypto_kdf108_ctr_generate); +int ret; +if (IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return 0 ; + + ret =kdf_test (&kdf_ctr_hmac_sha256_tv_template[0 ],"hmac(sha256)" , + crypto_kdf108_setkey, crypto_kdf108_ctr_generate);if (ret) {if (fips_enabled) panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
Signature · 1 line
2.38.1
PATCH v3 5/6 crypto: silence noisy kdf_sp800108 self-test
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
Make the kdf_sp800108 self-test only print a message on success when fips_enabled, so that it's consistent with testmgr.c and doesn't spam the kernel log with a message that isn't really important.
--- crypto/kdf_sp800108.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
crypto/kdf_sp800108.c +1 -1
@@ -140,7 +140,7 @@ static int __init crypto_kdf108_init(void)WARN (1 ,"alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n" , ret); - }else { + }else if (fips_enabled) {pr_info ("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n" ); }
Signature · 1 line
2.38.1
PATCH v3 6/6 crypto: compile out test-related algboss code when tests disabled
In reply to Eric BiggersFrom: Eric Biggers <[email protected]>
When CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is set, the code in algboss.c that handles CRYPTO_MSG_ALG_REGISTER is unnecessary, so make it be compiled out.
--- crypto/algboss.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
crypto/algboss.c +4 -5
@@ -175,11 +175,7 @@ static int cryptomgr_test(void *data) {struct crypto_test_param *param = data;u32 type = param->type; -int err =0 ; - -#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS - goto skiptest; -#endif +int err; err =alg_test (param->driver, param->alg, type,CRYPTO_ALG_TESTED );
@@ -194,6 +190,9 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg)struct task_struct *thread;struct crypto_test_param *param; +if (IS_ENABLED (CONFIG_CRYPTO_MANAGER_DISABLE_TESTS )) +return NOTIFY_DONE ; +if (!try_module_get (THIS_MODULE )) goto err;
Signature · 1 line
2.38.1
Herbert Xu
maintainerQuoted 27 lines · Eric Biggers
This patchset makes it so that the self-test code doesn't still slow things down when self-tests are disabled via the kconfig.
It also optimizes the registration of "internal" algorithms and silences a noisy log message.
Changed in v3:
- Made sureCRYPTO_MSG_ALG_LOADEDstill always gets sent.
- Fixed a race condition withlarval->test_started.
- UsedIS_ENABLED()in a couple places to avoid #ifdefs.Eric Biggers (6):
crypto: optimize algorithm registration when self-tests disabled
crypto: optimize registration of internal algorithms
crypto: compile outcrypto_boot_test_finishedwhen tests disabled
crypto: skipkdf_sp800108self-test when tests disabled
crypto: silence noisykdf_sp800108self-test
crypto: compile out test-related algboss code when tests disabledcrypto/algapi.c | 160 ++++++++++++++++++++++++------------------ crypto/algboss.c | 22 ++---- crypto/api.c | 11 ++- crypto/internal.h | 20 +++++- crypto/kdf_sp800108.c | 10 ++- 5 files changed, 127 insertions(+), 96 deletions(-)
Quoted 1 line
base-commit:
557ffd5a4726f8b6f0dd1d4b632ae02c1c063233
All applied. Thanks.
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt