crypto: Make memcpy_sglist usable for Crypto API
Subsystems (2)
Herbert Xu
maintainercover crypto: Make memcpy_sglist usable for Crypto API
I added memcpy_sglist for the simple SG lists used by zswap. It is quite similar in terms functionality to the skcipher_null algorithm.
However, before memcpy_sglist can replace skcipher_null, it needs to handle funky SG lists generated by IPsec. The SG lists are distinct, but contain identical subsets.
This patch series adds this functionality to memcpy_sglist by remodelling it on top of skcipher_walk instead of scatterwalk.
As a prerequisite, skcipher_walk has been abstracted out of skcipher and moved into scatterwalk.
Herbert Xu (2):
crypto: ccp - Include crypto/utils.h for crypto_memneq
crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist
crypto/scatterwalk.c | 274 +++++++++++++++++++++++++++-- crypto/skcipher.c | 261 +-------------------------- drivers/crypto/ccp/ccp-ops.c | 11 +- include/crypto/algapi.h | 12 -- include/crypto/internal/skcipher.h | 48 +---- include/crypto/scatterwalk.h | 65 ++++++- 6 files changed, 335 insertions(+), 336 deletions(-)
base-commit: 432f98cf56cf1e0f812f79a713847b37ae86cf31
Signature · 1 line
2.39.5
Herbert Xu
maintainerPATCH 1/2 crypto: ccp - Include crypto/utils.h for crypto_memneq
Include crypto/utils.h to get crypto_memneq rather than relying on random inclusions.
--- drivers/crypto/ccp/ccp-ops.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
drivers/crypto/ccp/ccp-ops.c +6 -5
@@ -8,13 +8,14 @@* Author: Gary R Hook <[email protected]> */ -#include <linux/dma-mapping.h> -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/interrupt.h> -#include <crypto/scatterwalk.h> #include <crypto/des.h> +#include <crypto/scatterwalk.h> +#include <crypto/utils.h> #include <linux/ccp.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> #include "ccp-dev.h"
Signature · 1 line
2.39.5
Herbert Xu
maintainerPATCH 2/2 crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist
In reply to Herbert XuMove the generic part of skcipher walk into scatterwalk, and use it to implement memcpy_sglist.
This makes memcpy_sglist do the right thing when two distinct SG lists contain identical subsets (e.g., the AD part of AEAD).
--- crypto/scatterwalk.c | 274 +++++++++++++++++++++++++++-- crypto/skcipher.c | 261 +-------------------------- include/crypto/algapi.h | 12 -- include/crypto/internal/skcipher.h | 48 +---- include/crypto/scatterwalk.h | 65 ++++++- 5 files changed, 329 insertions(+), 331 deletions(-)
crypto/scatterwalk.c +259 -15
@@ -10,10 +10,25 @@*/ #include <crypto/scatterwalk.h> +#include <linux/crypto.h> +#include <linux/errno.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/scatterlist.h> +#include <linux/slab.h> + +enum { +SKCIPHER_WALK_SLOW =1 <<0 , +SKCIPHER_WALK_COPY =1 <<1 , +SKCIPHER_WALK_DIFF =1 <<2 , +SKCIPHER_WALK_SLEEP =1 <<3 , +}; + +static inline gfp_t skcipher_walk_gfp (struct skcipher_walk *walk) +{ +return walk->flags &SKCIPHER_WALK_SLEEP ?GFP_KERNEL :GFP_ATOMIC ; +}void scatterwalk_skip (struct scatter_walk *walk,unsigned int nbytes) {
@@ -89,27 +104,23 @@ EXPORT_SYMBOL_GPL(memcpy_to_sglist);void memcpy_sglist (struct scatterlist *dst,struct scatterlist *src,unsigned int nbytes) { -struct scatter_walk swalk; -struct scatter_walk dwalk; +struct skcipher_walk walk = {};if (unlikely (nbytes ==0 ))/* in case sg == NULL */ return ; -scatterwalk_start (&swalk, src); -scatterwalk_start (&dwalk, dst); + walk.total = nbytes; +scatterwalk_start (&walk.in, src); +scatterwalk_start (&walk.out, dst); + +skcipher_walk_first (&walk, true);do { -unsigned int slen, dlen; -unsigned int len; - - slen =scatterwalk_next (&swalk, nbytes); - dlen =scatterwalk_next (&dwalk, nbytes); - len =min (slen, dlen); -memcpy (dwalk.addr, swalk.addr, len); -scatterwalk_done_dst (&dwalk, len); -scatterwalk_done_src (&swalk, len); - nbytes -= len; - }while (nbytes); +if (walk.src.virt.addr != walk.dst.virt.addr) +memcpy (walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes); +skcipher_walk_done (&walk,0 ); + }while (walk.nbytes); }EXPORT_SYMBOL_GPL (memcpy_sglist);
@@ -135,3 +146,236 @@ struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],return dst; }EXPORT_SYMBOL_GPL (scatterwalk_ffwd); + +static int skcipher_next_slow (struct skcipher_walk *walk,unsigned int bsize) +{ +unsigned alignmask = walk->alignmask; +unsigned n; +void *buffer; + +if (!walk->buffer) + walk->buffer = walk->page; + buffer = walk->buffer; +if (!buffer) { +/* Min size for a buffer of bsize bytes aligned to alignmask */ + n = bsize + (alignmask & ~(crypto_tfm_ctx_alignment () -1 )); + + buffer =kzalloc (n,skcipher_walk_gfp (walk)); +if (!buffer) +return skcipher_walk_done (walk, -ENOMEM ); + walk->buffer = buffer; + } + + buffer =PTR_ALIGN (buffer, alignmask +1 ); +memcpy_from_scatterwalk (buffer, &walk->in, bsize); + walk->out.__addr = buffer; + walk->in.__addr = walk->out.addr; + + walk->nbytes = bsize; + walk->flags |=SKCIPHER_WALK_SLOW ; + +return 0 ; +} + +static int skcipher_next_copy (struct skcipher_walk *walk) +{ +void *tmp = walk->page; + +scatterwalk_map (&walk->in); +memcpy (tmp, walk->in.addr, walk->nbytes); +scatterwalk_unmap (&walk->in); +/* +* walk->in is advanced later when the number of bytes actually +* processed (which might be less than walk->nbytes) is known. +*/ + + walk->in.__addr = tmp; + walk->out.__addr = tmp; +return 0 ; +} + +static int skcipher_next_fast (struct skcipher_walk *walk) +{ +unsigned long diff; + + diff =offset_in_page (walk->in.offset) - +offset_in_page (walk->out.offset); + diff |= (u8 *)(sg_page (walk->in.sg) + (walk->in.offset >>PAGE_SHIFT )) - + (u8 *)(sg_page (walk->out.sg) + (walk->out.offset >>PAGE_SHIFT )); + +scatterwalk_map (&walk->out); + walk->in.__addr = walk->out.__addr; + +if (diff) { + walk->flags |=SKCIPHER_WALK_DIFF ; +scatterwalk_map (&walk->in); + } + +return 0 ; +} + +static int skcipher_walk_next (struct skcipher_walk *walk) +{ +unsigned int bsize; +unsigned int n; + + n = walk->total; + bsize =min (walk->stride,max (n, walk->blocksize)); + n =scatterwalk_clamp (&walk->in, n); + n =scatterwalk_clamp (&walk->out, n); + +if (unlikely (n < bsize)) { +if (unlikely (walk->total < walk->blocksize)) +return skcipher_walk_done (walk, -EINVAL ); + +slow_path: +return skcipher_next_slow (walk, bsize); + } + walk->nbytes = n; + +if (unlikely ((walk->in.offset | walk->out.offset) & walk->alignmask)) { +if (!walk->page) { +gfp_t gfp =skcipher_walk_gfp (walk); + + walk->page = (void *)__get_free_page (gfp); +if (!walk->page) + goto slow_path; + } + walk->flags |=SKCIPHER_WALK_COPY ; +return skcipher_next_copy (walk); + } + +return skcipher_next_fast (walk); +} + +static int skcipher_copy_iv (struct skcipher_walk *walk) +{ +unsigned alignmask = walk->alignmask; +unsigned ivsize = walk->ivsize; +unsigned aligned_stride =ALIGN (walk->stride, alignmask +1 ); +unsigned size; +u8 *iv; + +/* Min size for a buffer of stride + ivsize, aligned to alignmask */ + size = aligned_stride + ivsize + + (alignmask & ~(crypto_tfm_ctx_alignment () -1 )); + + walk->buffer =kmalloc (size,skcipher_walk_gfp (walk)); +if (!walk->buffer) +return -ENOMEM ; + + iv =PTR_ALIGN (walk->buffer, alignmask +1 ) + aligned_stride; + + walk->iv =memcpy (iv, walk->iv, walk->ivsize); +return 0 ; +} + +int skcipher_walk_first (struct skcipher_walk *walk,bool atomic) +{ +if (WARN_ON_ONCE (in_hardirq ())) +return -EDEADLK ; + + walk->flags = atomic ?0 :SKCIPHER_WALK_SLEEP ; + + walk->buffer =NULL ; +if (unlikely (((unsigned long )walk->iv & walk->alignmask))) { +int err =skcipher_copy_iv (walk); +if (err) +return err; + } + + walk->page =NULL ; + +return skcipher_walk_next (walk); +} +EXPORT_SYMBOL_GPL (skcipher_walk_first); + +/** +* skcipher_walk_done() - finish one step of a skcipher_walk +* @walk: the skcipher_walk +* @res: number of bytes *not* processed (>= 0) from walk->nbytes, +* or a -errno value to terminate the walk due to an error +* +* This function cleans up after one step of walking through the source and +* destination scatterlists, and advances to the next step if applicable. +* walk->nbytes is set to the number of bytes available in the next step, +* walk->total is set to the new total number of bytes remaining, and +* walk->{src,dst}.virt.addr is set to the next pair of data pointers. If there +* is no more data, or if an error occurred (i.e. -errno return), then +* walk->nbytes and walk->total are set to 0 and all resources owned by the +* skcipher_walk are freed. +* +* Return: 0 or a -errno value. If @res was a -errno value then it will be +* returned, but other errors may occur too. +*/ +int skcipher_walk_done (struct skcipher_walk *walk,int res) +{ +unsigned int n = walk->nbytes;/* num bytes processed this step */ +unsigned int total =0 ;/* new total remaining */ + +if (!n) + goto finish; + +if (likely (res >=0 )) { + n -= res;/* subtract num bytes *not* processed */ + total = walk->total - n; + } + +if (likely (!(walk->flags & (SKCIPHER_WALK_SLOW | +SKCIPHER_WALK_COPY | +SKCIPHER_WALK_DIFF )))) { +scatterwalk_advance (&walk->in, n); + }else if (walk->flags &SKCIPHER_WALK_DIFF ) { +scatterwalk_done_src (&walk->in, n); + }else if (walk->flags &SKCIPHER_WALK_COPY ) { +scatterwalk_advance (&walk->in, n); +scatterwalk_map (&walk->out); +memcpy (walk->out.addr, walk->page, n); + }else {/* SKCIPHER_WALK_SLOW */ +if (res >0 ) { +/* +* Didn't process all bytes. Either the algorithm is +* broken, or this was the last step and it turned out +* the message wasn't evenly divisible into blocks but +* the algorithm requires it. +*/ + res = -EINVAL ; + total =0 ; + }else +memcpy_to_scatterwalk (&walk->out, walk->out.addr, n); + goto dst_done; + } + +scatterwalk_done_dst (&walk->out, n); +dst_done: + +if (res >0 ) + res =0 ; + + walk->total = total; + walk->nbytes =0 ; + +if (total) { +if (walk->flags &SKCIPHER_WALK_SLEEP ) +cond_resched (); + walk->flags &= ~(SKCIPHER_WALK_SLOW |SKCIPHER_WALK_COPY | +SKCIPHER_WALK_DIFF ); +return skcipher_walk_next (walk); + } + +finish: +/* Short-circuit for the common/fast path. */ +if (!((unsigned long )walk->buffer | (unsigned long )walk->page)) + goto out; + +if (walk->iv != walk->oiv) +memcpy (walk->oiv, walk->iv, walk->ivsize); +if (walk->buffer != walk->page) +kfree (walk->buffer); +if (walk->page) +free_page ((unsigned long )walk->page); + +out: +return res; +} +EXPORT_SYMBOL_GPL (skcipher_walk_done);
crypto/skcipher.c +6 -255
305 lines not on this page · raw message
include/crypto/algapi.h +0 -12
@@ -107,18 +107,6 @@ struct crypto_queue {unsigned int max_qlen; }; -struct scatter_walk { -/* Must be the first member, see struct skcipher_walk. */ -union { -void *const addr; - -/* Private API field, do not touch. */ -union crypto_no_such_thing *__addr; - }; -struct scatterlist *sg; -unsigned int offset; -}; -struct crypto_attr_alg {char name[CRYPTO_MAX_ALG_NAME ]; };
include/crypto/internal/skcipher.h +1 -47
72 lines not on this page · raw message
include/crypto/scatterwalk.h +63 -2
78 lines not on this page · raw message
Signature · 1 line
2.39.5
Re: [PATCH 2/2] crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist
Quoted 7 lines · Herbert Xu
Move the generic part of skcipher walk into scatterwalk, and use it to implement
memcpy_sglist.This makes
memcpy_sglistdo the right thing when two distinct SG lists contain identical subsets (e.g., the AD part of AEAD).Signed-off-by Herbert Xu <[email protected]>
I think it would end up faster and simpler to just implement the copy from first principles instead of trying to share the code with skcipher_walk. There is way too much stuff that skcipher_walk has to handle that isn't relevant for a simple copy.
- Eric
Herbert Xu
maintainerRe: [PATCH 2/2] crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist
Quoted 5 lines · Eric Biggers
I think it would end up faster and simpler to just implement the copy from first principles instead of trying to share the code with
skcipher_walk. There is way too much stuff thatskcipher_walkhas to handle that isn't relevant for a simple copy.
Sure, please feel free to improve this function further. I just needed something that works for chacha20poly1305 and this was the easiest way to get it done.
Thanks,
Signature · 3 lines
Email: Herbert Xu <[email protected]> Home Page: http://gondor.apana.org.au/~herbert/ PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt