Skip to contentSkip to conversation

crypto: Make memcpy_sglist usable for Crypto API

PATCH In review Herbert Xu 2 participants 5 messages Apr 27, 2025

Subsystems (2)
lore
Share & tools
lore

Herbert Xu

author maintainer
#0 lore raw

cover crypto: Make memcpy_sglist usable for Crypto API

I added memcpy_sglist for the simple SG lists used by zswap. It is quite similar in terms functionality to the skcipher_null algorithm.

However, before memcpy_sglist can replace skcipher_null, it needs to handle funky SG lists generated by IPsec. The SG lists are distinct, but contain identical subsets.

This patch series adds this functionality to memcpy_sglist by remodelling it on top of skcipher_walk instead of scatterwalk.

As a prerequisite, skcipher_walk has been abstracted out of skcipher and moved into scatterwalk.

Herbert Xu (2):
crypto: ccp - Include crypto/utils.h for crypto_memneq
crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist

 crypto/scatterwalk.c               | 274 +++++++++++++++++++++++++++--
 crypto/skcipher.c                  | 261 +--------------------------
 drivers/crypto/ccp/ccp-ops.c       |  11 +-
 include/crypto/algapi.h            |  12 --
 include/crypto/internal/skcipher.h |  48 +----
 include/crypto/scatterwalk.h       |  65 ++++++-
 6 files changed, 335 insertions(+), 336 deletions(-)

base-commit: 432f98cf56cf1e0f812f79a713847b37ae86cf31

Signature · 1 line
2.39.5
author maintainer
#1 lore raw

PATCH 1/2 crypto: ccp - Include crypto/utils.h for crypto_memneq

Include crypto/utils.h to get crypto_memneq rather than relying on random inclusions.

---
 drivers/crypto/ccp/ccp-ops.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
drivers/crypto/ccp/ccp-ops.c +6 -5
index cb8e99936abb..109b5aef4034 100644
@@ -8,13 +8,14 @@
  * Author: Gary R Hook <[email protected]>
  */
 
-#include <linux/dma-mapping.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/interrupt.h>
-#include <crypto/scatterwalk.h>
 #include <crypto/des.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/utils.h>
 #include <linux/ccp.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 
 #include "ccp-dev.h"
 
Signature · 1 line
2.39.5
author maintainer
#2 lore raw

PATCH 2/2 crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist

In reply to Herbert Xu

Move the generic part of skcipher walk into scatterwalk, and use it to implement memcpy_sglist.

This makes memcpy_sglist do the right thing when two distinct SG lists contain identical subsets (e.g., the AD part of AEAD).

---
 crypto/scatterwalk.c               | 274 +++++++++++++++++++++++++++--
 crypto/skcipher.c                  | 261 +--------------------------
 include/crypto/algapi.h            |  12 --
 include/crypto/internal/skcipher.h |  48 +----
 include/crypto/scatterwalk.h       |  65 ++++++-
 5 files changed, 329 insertions(+), 331 deletions(-)
crypto/scatterwalk.c +259 -15
index 8225801488d5..1d010e2a1b1a 100644
@@ -10,10 +10,25 @@
  */
 
 #include <crypto/scatterwalk.h>
+#include <linux/crypto.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
+#include <linux/slab.h>
+
+enum {
+	SKCIPHER_WALK_SLOW = 1 << 0,
+	SKCIPHER_WALK_COPY = 1 << 1,
+	SKCIPHER_WALK_DIFF = 1 << 2,
+	SKCIPHER_WALK_SLEEP = 1 << 3,
+};
+
+static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk)
+{
+	return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
+}
 
 void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes)
 {
@@ -89,27 +104,23 @@ EXPORT_SYMBOL_GPL(memcpy_to_sglist);
 void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src,
 		   unsigned int nbytes)
 {
-	struct scatter_walk swalk;
-	struct scatter_walk dwalk;
+	struct skcipher_walk walk = {};
 
 	if (unlikely(nbytes == 0)) /* in case sg == NULL */
 		return;
 
-	scatterwalk_start(&swalk, src);
-	scatterwalk_start(&dwalk, dst);
+	walk.total = nbytes;
 
+	scatterwalk_start(&walk.in, src);
+	scatterwalk_start(&walk.out, dst);
+
+	skcipher_walk_first(&walk, true);
 	do {
-		unsigned int slen, dlen;
-		unsigned int len;
-
-		slen = scatterwalk_next(&swalk, nbytes);
-		dlen = scatterwalk_next(&dwalk, nbytes);
-		len = min(slen, dlen);
-		memcpy(dwalk.addr, swalk.addr, len);
-		scatterwalk_done_dst(&dwalk, len);
-		scatterwalk_done_src(&swalk, len);
-		nbytes -= len;
-	} while (nbytes);
+		if (walk.src.virt.addr != walk.dst.virt.addr)
+			memcpy(walk.dst.virt.addr, walk.src.virt.addr,
+			       walk.nbytes);
+		skcipher_walk_done(&walk, 0);
+	} while (walk.nbytes);
 }
 EXPORT_SYMBOL_GPL(memcpy_sglist);
 
@@ -135,3 +146,236 @@ struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2],
 	return dst;
 }
 EXPORT_SYMBOL_GPL(scatterwalk_ffwd);
+
+static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize)
+{
+	unsigned alignmask = walk->alignmask;
+	unsigned n;
+	void *buffer;
+
+	if (!walk->buffer)
+		walk->buffer = walk->page;
+	buffer = walk->buffer;
+	if (!buffer) {
+		/* Min size for a buffer of bsize bytes aligned to alignmask */
+		n = bsize + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
+
+		buffer = kzalloc(n, skcipher_walk_gfp(walk));
+		if (!buffer)
+			return skcipher_walk_done(walk, -ENOMEM);
+		walk->buffer = buffer;
+	}
+
+	buffer = PTR_ALIGN(buffer, alignmask + 1);
+	memcpy_from_scatterwalk(buffer, &walk->in, bsize);
+	walk->out.__addr = buffer;
+	walk->in.__addr = walk->out.addr;
+
+	walk->nbytes = bsize;
+	walk->flags |= SKCIPHER_WALK_SLOW;
+
+	return 0;
+}
+
+static int skcipher_next_copy(struct skcipher_walk *walk)
+{
+	void *tmp = walk->page;
+
+	scatterwalk_map(&walk->in);
+	memcpy(tmp, walk->in.addr, walk->nbytes);
+	scatterwalk_unmap(&walk->in);
+	/*
+	 * walk->in is advanced later when the number of bytes actually
+	 * processed (which might be less than walk->nbytes) is known.
+	 */
+
+	walk->in.__addr = tmp;
+	walk->out.__addr = tmp;
+	return 0;
+}
+
+static int skcipher_next_fast(struct skcipher_walk *walk)
+{
+	unsigned long diff;
+
+	diff = offset_in_page(walk->in.offset) -
+	       offset_in_page(walk->out.offset);
+	diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) -
+		(u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT));
+
+	scatterwalk_map(&walk->out);
+	walk->in.__addr = walk->out.__addr;
+
+	if (diff) {
+		walk->flags |= SKCIPHER_WALK_DIFF;
+		scatterwalk_map(&walk->in);
+	}
+
+	return 0;
+}
+
+static int skcipher_walk_next(struct skcipher_walk *walk)
+{
+	unsigned int bsize;
+	unsigned int n;
+
+	n = walk->total;
+	bsize = min(walk->stride, max(n, walk->blocksize));
+	n = scatterwalk_clamp(&walk->in, n);
+	n = scatterwalk_clamp(&walk->out, n);
+
+	if (unlikely(n < bsize)) {
+		if (unlikely(walk->total < walk->blocksize))
+			return skcipher_walk_done(walk, -EINVAL);
+
+slow_path:
+		return skcipher_next_slow(walk, bsize);
+	}
+	walk->nbytes = n;
+
+	if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) {
+		if (!walk->page) {
+			gfp_t gfp = skcipher_walk_gfp(walk);
+
+			walk->page = (void *)__get_free_page(gfp);
+			if (!walk->page)
+				goto slow_path;
+		}
+		walk->flags |= SKCIPHER_WALK_COPY;
+		return skcipher_next_copy(walk);
+	}
+
+	return skcipher_next_fast(walk);
+}
+
+static int skcipher_copy_iv(struct skcipher_walk *walk)
+{
+	unsigned alignmask = walk->alignmask;
+	unsigned ivsize = walk->ivsize;
+	unsigned aligned_stride = ALIGN(walk->stride, alignmask + 1);
+	unsigned size;
+	u8 *iv;
+
+	/* Min size for a buffer of stride + ivsize, aligned to alignmask */
+	size = aligned_stride + ivsize +
+	       (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
+
+	walk->buffer = kmalloc(size, skcipher_walk_gfp(walk));
+	if (!walk->buffer)
+		return -ENOMEM;
+
+	iv = PTR_ALIGN(walk->buffer, alignmask + 1) + aligned_stride;
+
+	walk->iv = memcpy(iv, walk->iv, walk->ivsize);
+	return 0;
+}
+
+int skcipher_walk_first(struct skcipher_walk *walk, bool atomic)
+{
+	if (WARN_ON_ONCE(in_hardirq()))
+		return -EDEADLK;
+
+	walk->flags = atomic ? 0 : SKCIPHER_WALK_SLEEP;
+
+	walk->buffer = NULL;
+	if (unlikely(((unsigned long)walk->iv & walk->alignmask))) {
+		int err = skcipher_copy_iv(walk);
+		if (err)
+			return err;
+	}
+
+	walk->page = NULL;
+
+	return skcipher_walk_next(walk);
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_first);
+
+/**
+ * skcipher_walk_done() - finish one step of a skcipher_walk
+ * @walk: the skcipher_walk
+ * @res: number of bytes *not* processed (>= 0) from walk->nbytes,
+ *	 or a -errno value to terminate the walk due to an error
+ *
+ * This function cleans up after one step of walking through the source and
+ * destination scatterlists, and advances to the next step if applicable.
+ * walk->nbytes is set to the number of bytes available in the next step,
+ * walk->total is set to the new total number of bytes remaining, and
+ * walk->{src,dst}.virt.addr is set to the next pair of data pointers.  If there
+ * is no more data, or if an error occurred (i.e. -errno return), then
+ * walk->nbytes and walk->total are set to 0 and all resources owned by the
+ * skcipher_walk are freed.
+ *
+ * Return: 0 or a -errno value.  If @res was a -errno value then it will be
+ *	   returned, but other errors may occur too.
+ */
+int skcipher_walk_done(struct skcipher_walk *walk, int res)
+{
+	unsigned int n = walk->nbytes; /* num bytes processed this step */
+	unsigned int total = 0; /* new total remaining */
+
+	if (!n)
+		goto finish;
+
+	if (likely(res >= 0)) {
+		n -= res; /* subtract num bytes *not* processed */
+		total = walk->total - n;
+	}
+
+	if (likely(!(walk->flags & (SKCIPHER_WALK_SLOW |
+				    SKCIPHER_WALK_COPY |
+				    SKCIPHER_WALK_DIFF)))) {
+		scatterwalk_advance(&walk->in, n);
+	} else if (walk->flags & SKCIPHER_WALK_DIFF) {
+		scatterwalk_done_src(&walk->in, n);
+	} else if (walk->flags & SKCIPHER_WALK_COPY) {
+		scatterwalk_advance(&walk->in, n);
+		scatterwalk_map(&walk->out);
+		memcpy(walk->out.addr, walk->page, n);
+	} else { /* SKCIPHER_WALK_SLOW */
+		if (res > 0) {
+			/*
+			 * Didn't process all bytes.  Either the algorithm is
+			 * broken, or this was the last step and it turned out
+			 * the message wasn't evenly divisible into blocks but
+			 * the algorithm requires it.
+			 */
+			res = -EINVAL;
+			total = 0;
+		} else
+			memcpy_to_scatterwalk(&walk->out, walk->out.addr, n);
+		goto dst_done;
+	}
+
+	scatterwalk_done_dst(&walk->out, n);
+dst_done:
+
+	if (res > 0)
+		res = 0;
+
+	walk->total = total;
+	walk->nbytes = 0;
+
+	if (total) {
+		if (walk->flags & SKCIPHER_WALK_SLEEP)
+			cond_resched();
+		walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY |
+				 SKCIPHER_WALK_DIFF);
+		return skcipher_walk_next(walk);
+	}
+
+finish:
+	/* Short-circuit for the common/fast path. */
+	if (!((unsigned long)walk->buffer | (unsigned long)walk->page))
+		goto out;
+
+	if (walk->iv != walk->oiv)
+		memcpy(walk->oiv, walk->iv, walk->ivsize);
+	if (walk->buffer != walk->page)
+		kfree(walk->buffer);
+	if (walk->page)
+		free_page((unsigned long)walk->page);
+
+out:
+	return res;
+}
+EXPORT_SYMBOL_GPL(skcipher_walk_done);
crypto/skcipher.c +6 -255

305 lines not on this page · raw message

include/crypto/algapi.h +0 -12
index 6999e10ea09e..f5f730969d72 100644
@@ -107,18 +107,6 @@ struct crypto_queue {
 	unsigned int max_qlen;
 };
 
-struct scatter_walk {
-	/* Must be the first member, see struct skcipher_walk. */
-	union {
-		void *const addr;
-
-		/* Private API field, do not touch. */
-		union crypto_no_such_thing *__addr;
-	};
-	struct scatterlist *sg;
-	unsigned int offset;
-};
-
 struct crypto_attr_alg {
 	char name[CRYPTO_MAX_ALG_NAME];
 };
include/crypto/internal/skcipher.h +1 -47

72 lines not on this page · raw message

include/crypto/scatterwalk.h +63 -2

78 lines not on this page · raw message

Signature · 1 line
2.39.5
#3 lore raw

Re: [PATCH 2/2] crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist

Quoted 7 lines · Herbert Xu

Move the generic part of skcipher walk into scatterwalk, and use it to implement memcpy_sglist.

This makes memcpy_sglist do the right thing when two distinct SG lists contain identical subsets (e.g., the AD part of AEAD).

I think it would end up faster and simpler to just implement the copy from first principles instead of trying to share the code with skcipher_walk. There is way too much stuff that skcipher_walk has to handle that isn't relevant for a simple copy.

- Eric

author maintainer
#4 lore raw

Re: [PATCH 2/2] crypto: scatterwalk - Move skcipher walk and use it for memcpy_sglist

Quoted 5 lines · Eric Biggers

I think it would end up faster and simpler to just implement the copy from first principles instead of trying to share the code with skcipher_walk. There is way too much stuff that skcipher_walk has to handle that isn't relevant for a simple copy.

Sure, please feel free to improve this function further. I just needed something that works for chacha20poly1305 and this was the easiest way to get it done.

Thanks,

Signature · 3 lines
Email: Herbert Xu <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt