Adding a generic scatterlist eater: generic scatterwalk
PATCH 2/4 Adding a generic scatterlist eater: generic scatterwalk
This generic scatterwalker is an abstraction of the crypt(..) function. It allows data to be processed by a processing function from an arbitrary number of scatterlist which constitute the arguments to the function.
I felt the need to add a large comment explaining the use of this function. This function is surely useful not only for the cryptoapi subsystem. Maybe we should put it into a more common place. Of course, I replace the former crypt clients with code for scatterwalk_walker_generic. While doing this, I replaced the CBC logic for a short initial vector ring-buffer. This reduces the number of IV memcpy's to 1 at most.
This patch is the one responsible for the performance increase mentioned in http://lkml.org/lkml/2005/1/19/191
crypto/cipher.c +122 -102
@@ -39,91 +39,49 @@ } -/* -* Generic encrypt/decrypt wrapper for ciphers, handles operations across -* multiple page boundaries by using temporary blocks. In user context, -* the kernel is given a chance to schedule us once per block. -*/ -static int crypt (struct crypto_tfm *tfm, -struct scatterlist *dst, -struct scatterlist *src, -unsigned int nbytes,cryptfn_t crfn, -procfn_t prfn,int enc,void *info) -{ -struct scatter_walk walk_in, walk_out; -const unsigned int bsize =crypto_tfm_alg_blocksize (tfm); -u8 tmp_src[bsize]; -u8 tmp_dst[bsize]; - -if (!nbytes) -return 0 ; - -if (nbytes % bsize) { - tfm->crt_flags |=CRYPTO_TFM_RES_BAD_BLOCK_LEN ; -return -EINVAL ; - } - -scatterwalk_start (&walk_in, src); -scatterwalk_start (&walk_out, dst); - -for (;;) { -u8 *src_p, *dst_p; -int in_place; - -scatterwalk_map (&walk_in,0 ); -scatterwalk_map (&walk_out,1 ); - src_p =scatterwalk_whichbuf (&walk_in, bsize, tmp_src); - dst_p =scatterwalk_whichbuf (&walk_out, bsize, tmp_dst); - in_place =scatterwalk_samebuf (&walk_in, &walk_out, - src_p, dst_p); - - nbytes -= bsize; - -scatterwalk_copychunks (src_p, &walk_in, bsize,0 ); - -prfn (tfm, dst_p, src_p, crfn, enc, info, in_place); - -scatterwalk_done (&walk_in,0 , nbytes); - -scatterwalk_copychunks (dst_p, &walk_out, bsize,1 ); -scatterwalk_done (&walk_out,1 , nbytes); - -if (!nbytes) -return 0 ; - -crypto_yield (tfm); - } -} - -static void cbc_process (struct crypto_tfm *tfm,u8 *dst,u8 *src, -cryptfn_t fn,int enc,void *info,int in_place) -{ -u8 *iv = info; +struct cbc_process_priv { +struct crypto_tfm *tfm; +int enc; +cryptfn_t *crfn; +u8 *curIV; +u8 *nextIV; +}; + +static void cbc_process_gw (void *_priv,int nsg,void **buf) +{ +struct cbc_process_priv *priv = (struct cbc_process_priv *)_priv; +u8 *iv = priv->curIV; +struct crypto_tfm *tfm = priv->tfm; +int bsize =crypto_tfm_alg_blocksize (tfm); +u8 *dst = buf[0 ]; +u8 *src = buf[1 ];/* Null encryption */ if (!iv)return ; - -if (enc) { + +if (priv->enc) { tfm->crt_u.cipher.cit_xor_block (iv, src); -fn (crypto_tfm_ctx (tfm), dst, iv); -memcpy (iv, dst,crypto_tfm_alg_blocksize (tfm)); + priv->crfn (crypto_tfm_ctx (tfm), dst, iv); +memcpy (iv, dst, bsize); }else { -u8 stack[in_place ?crypto_tfm_alg_blocksize (tfm) :0 ]; -u8 *buf = in_place ? stack : dst; - -fn (crypto_tfm_ctx (tfm), buf, src); - tfm->crt_u.cipher.cit_xor_block (buf, iv); -memcpy (iv, src,crypto_tfm_alg_blocksize (tfm)); -if (buf != dst) -memcpy (dst, buf,crypto_tfm_alg_blocksize (tfm)); +memcpy (priv->nextIV,src,bsize); + priv->crfn (crypto_tfm_ctx (tfm), dst, src); + tfm->crt_u.cipher.cit_xor_block (dst, iv); + priv->curIV = priv->nextIV; + priv->nextIV = iv; } } -static void ecb_process (struct crypto_tfm *tfm,u8 *dst,u8 *src, -cryptfn_t fn,int enc,void *info,int in_place) +struct ecb_process_priv { +struct crypto_tfm *tfm; +cryptfn_t *crfn; +}; + +static void ecb_process_gw (void *_priv,int nsg,void **buf) { -fn (crypto_tfm_ctx (tfm), dst, src); +struct ecb_process_priv *priv = (struct ecb_process_priv *)_priv; + priv->crfn (crypto_tfm_ctx (priv->tfm), buf[0 ], buf[1 ]); }static int setkey (struct crypto_tfm *tfm,const u8 *key,unsigned int keylen)
@@ -142,9 +100,25 @@struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - ecb_process, 1, NULL); + int bsize = crypto_tfm_alg_blocksize(tfm); + +struct walk_info ecb_info[2 ] = { + [0 ].sg = dst, + [0 ].stepsize = bsize, + [0 ].ioflag =1 , + [0 ].buf = (char [bsize]){}, + [1 ].sg = src, + [1 ].stepsize = bsize, + [1 ].ioflag =0 , + [1 ].buf = (char [bsize]){}, + }; + +struct ecb_process_priv priv = { + .tfm = tfm, + .crfn = tfm->__crt_alg->cra_cipher.cia_encrypt, + }; + +return scatterwalk_walker_generic (ecb_process_gw, &priv, nbytes/bsize,2 , ecb_info); } static int ecb_decrypt(struct crypto_tfm *tfm,
@@ -152,29 +126,79 @@struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - ecb_process, 1, NULL); + int bsize = crypto_tfm_alg_blocksize(tfm); + +struct walk_info ecb_info[2 ] = { + [0 ].sg = dst, + [0 ].stepsize = bsize, + [0 ].ioflag =1 , + [0 ].buf = (char [bsize]){}, + [1 ].sg = src, + [1 ].stepsize = bsize, + [1 ].ioflag =0 , + [1 ].buf = (char [bsize]){}, + }; +struct ecb_process_priv priv = { + .tfm = tfm, + .crfn = tfm->__crt_alg->cra_cipher.cia_decrypt, + }; + +return scatterwalk_walker_generic (ecb_process_gw, &priv, nbytes/bsize,2 , ecb_info); } -static int cbc_encrypt (struct crypto_tfm *tfm, -struct scatterlist *dst, -struct scatterlist *src, -unsigned int nbytes) +static int cbc_encrypt_iv (struct crypto_tfm *tfm, +struct scatterlist *dst, +struct scatterlist *src, +unsigned int nbytes,u8 *iv) { -return crypt (tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - cbc_process,1 , tfm->crt_cipher.cit_iv); +int bsize =crypto_tfm_alg_blocksize (tfm); +struct walk_info cbc_walk_info[2 ] = { + [0 ].sg = dst, + [0 ].stepsize = bsize, + [0 ].ioflag =1 , + [0 ].buf = (char [bsize]){}, + [1 ].sg = src, + [1 ].stepsize = bsize, + [1 ].ioflag =0 , + [1 ].buf = (char [bsize]){}, + }; +struct cbc_process_priv priv = { + .tfm = tfm, + .enc =1 , + .curIV = iv, + .crfn = tfm->__crt_alg->cra_cipher.cia_encrypt, + }; +return scatterwalk_walker_generic (cbc_process_gw, &priv, nbytes/bsize,2 , cbc_walk_info); } -static int cbc_encrypt_iv (struct crypto_tfm *tfm, +static int cbc_decrypt_iv (struct crypto_tfm *tfm,struct scatterlist *dst,struct scatterlist *src,unsigned int nbytes,u8 *iv) { -return crypt (tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - cbc_process,1 , iv); +int bsize =crypto_tfm_alg_blocksize (tfm); +struct walk_info cbc_walk_info[2 ] = { + [0 ].sg = dst, + [0 ].stepsize = bsize, + [0 ].ioflag =1 , + [0 ].buf = (char [bsize]){}, + [1 ].sg = src, + [1 ].stepsize = bsize, + [1 ].ioflag =0 , + [1 ].buf = (char [bsize]){}, + }; +int r; +struct cbc_process_priv priv = { + .tfm = tfm, + .enc =0 , + .curIV = iv, + .nextIV = (char [bsize]){}, + .crfn = tfm->__crt_alg->cra_cipher.cia_decrypt, + }; + r =scatterwalk_walker_generic (cbc_process_gw, &priv, nbytes/bsize,2 , cbc_walk_info); +if (priv.curIV != iv) +memcpy (iv,priv.curIV,bsize); +return r; } static int cbc_decrypt(struct crypto_tfm *tfm,
@@ -182,19 +206,15 @@struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - cbc_process, 0, tfm->crt_cipher.cit_iv); + return cbc_decrypt_iv(tfm, dst, src, nbytes, tfm->crt_cipher.cit_iv); } -static int cbc_decrypt_iv (struct crypto_tfm *tfm, -struct scatterlist *dst, -struct scatterlist *src, -unsigned int nbytes,u8 *iv) +static int cbc_encrypt (struct crypto_tfm *tfm, +struct scatterlist *dst, +struct scatterlist *src, +unsigned int nbytes) { -return crypt (tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - cbc_process,0 , iv); +return cbc_encrypt_iv (tfm, dst, src, nbytes, tfm->crt_cipher.cit_iv); } static int nocrypt(struct crypto_tfm *tfm,
crypto/api.c +1 -1
@@ -131,7 +131,7 @@struct crypto_tfm *tfm =NULL ;struct crypto_alg *alg;int tfm_size; - + alg =crypto_alg_mod_lookup (name);if (alg ==NULL ) goto out;
crypto/scatterwalk.h +24 -5
@@ -16,6 +16,7 @@#define _CRYPTO_SCATTERWALK_H#include <linux/mm.h> #include <asm/scatterlist.h> +#include <linux/pagemap.h> struct scatter_walk {struct scatterlist *sg;
@@ -26,6 +27,26 @@unsigned int offset; }; +struct walk_info { +struct scatterlist *sg; +int stepsize; +int ioflag; +char *buf; +struct scatter_walk sw; +int freebuf; + +}; + + +#define scatterwalk_needscratch (walk, nbytes) \ + ((nbytes) <= (walk)->len_this_page && \ + (((unsigned long)(walk)->data) & (PAGE_CACHE_SIZE - 1)) + (nbytes) <= \ + PAGE_CACHE_SIZE) \ + + +#define scatterwalk_whichbuf (walk,nbytes,scratch) \ + (scatterwalk_needscratch(walk,nbytes)?(walk)->data:(scratch)) +/* Define sg_next is an inline routine now in case we want to change scatterlist to a linked list later. */ static inline struct scatterlist *sg_next (struct scatterlist *sg)
@@ -42,10 +63,8 @@ walk_in->data == src_p && walk_out->data == dst_p; } -void *scatterwalk_whichbuf (struct scatter_walk *walk,unsigned int nbytes,void *scratch); -void scatterwalk_start (struct scatter_walk *walk,struct scatterlist *sg); -int scatterwalk_copychunks (void *buf,struct scatter_walk *walk,size_t nbytes,int out); -void scatterwalk_map (struct scatter_walk *walk,int out); -void scatterwalk_done (struct scatter_walk *walk,int out,int more); +typedef void (*sw_proc_func_t )(void *priv,int length,void **buflist); + +int scatterwalk_walker_generic (sw_proc_func_t function,void *priv,int steps,int nsl,struct walk_info *walk_info_l);#endif /* _CRYPTO_SCATTERWALK_H */
crypto/scatterwalk.c +114 -43
190 lines not on this page · raw message