Skip to contentSkip to conversation

dm-crypt: support relaxed memory alignment

PATCH RESEND Superseded Keith Busch 1 participant 3 messages Jan 14

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v3 cover dm-crypt: support relaxed memory alignment

From: Keith Busch <[email protected]>

Resending as the previous send was bad timing with the merge window.

Direct-io can support any memory alignment the hardware allows. Device mappers don't need to impose any software constraints on memory alignment, so this series removes one of those limitations the dm-crypt mapper.

Changes from v2:

* Don't change the default stacking limit to allow the relaxed memory alignment requirements; have the caller do it instead.

* Fixed scatterlist memory leaks when handling the case that can't use the inline scatterlist.

* Fixed segment boundary check to use the crypt_config rather than the lower level block device's dma_alignment, which may not be the same size as the cc->sector_size which was used before, or the newly enabled 4-byte alignment this patch set allows in certain circumstances.

Keith Busch (2):
dm-crypt: allow unaligned bio_vecs for direct io
dm-crypt: dynamic scatterlist for many segments

 drivers/md/dm-crypt.c | 114 ++++++++++++++++++++++++++++++++++--------
 drivers/md/dm-table.c |   1 +
 2 files changed, 94 insertions(+), 21 deletions(-)
Signature · 1 line
2.47.3
#1 lore raw

PATCH RESEND v3 1/2 dm-crypt: allow unaligned bio_vecs for direct io

From: Keith Busch <[email protected]>

Many storage devices can handle DMA for data that is not aligned to the sector block size. The block and filesystem layers have introduced updates to allow that kind of memory alignment flexibility when possible.

dm-crypt, however, currently constrains itself to aligned memory because it sends a single scatterlist element for the in/out list to the encrypt and decrypt algorithms. This forces applications that have unaligned data to copy through a bounce buffer, increasing CPU and memory utilization.

Use multiple scatterlist elements to relax the memory alignment requirement. To keep this simple, this more flexible constraint is enabled only for certain encryption and initialization vector types, specifically the ones that don't have additional use for the request base scatterlist elements beyond holding decrypted data.

---
 drivers/md/dm-crypt.c | 79 +++++++++++++++++++++++++++++++++----------
 drivers/md/dm-table.c |  1 +
 2 files changed, 62 insertions(+), 18 deletions(-)
drivers/md/dm-crypt.c +61 -18
index 5ef43231fe77f..5304954b4574b 100644
@@ -149,6 +149,7 @@ enum cipher_flags {
 	CRYPT_IV_LARGE_SECTORS,		/* Calculate IV from sector_size, not 512B sectors */
 	CRYPT_ENCRYPT_PREPROCESS,	/* Must preprocess data for encryption (elephant) */
 	CRYPT_KEY_MAC_SIZE_SET,		/* The integrity_key_size option was used */
+	CRYPT_DISCONTIGUOUS_SEGS,	/* Can use partial sector segments */
 };
 
 /*
@@ -214,6 +215,7 @@ struct crypt_config {
 	unsigned int key_extra_size; /* additional keys length */
 	unsigned int key_mac_size;   /* MAC key size for authenc(...) */
 
+	unsigned int io_alignment;
 	unsigned int integrity_tag_size;
 	unsigned int integrity_iv_size;
 	unsigned int used_tag_size;
@@ -1424,22 +1426,48 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
 	return r;
 }
 
+static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist *sg,
+			   struct bvec_iter *iter, struct bio *bio,
+			   int max_segs)
+{
+	unsigned int bytes = cc->sector_size;
+	struct bvec_iter tmp = *iter;
+	int segs, i = 0;
+
+	bio_advance_iter(bio, &tmp, bytes);
+	segs = tmp.bi_idx - iter->bi_idx + !!tmp.bi_bvec_done;
+	if (segs > max_segs)
+		return -EIO;
+
+	sg_init_table(sg, segs);
+	do {
+		struct bio_vec bv = mp_bvec_iter_bvec(bio->bi_io_vec, *iter);
+		int len = min(bytes, bv.bv_len);
+
+		/* Reject unexpected unaligned bio. */
+		if (unlikely((len | bv.bv_offset) & cc->io_alignment))
+			return -EIO;
+
+		sg_set_page(&sg[i++], bv.bv_page, len, bv.bv_offset);
+		bio_advance_iter_single(bio, iter, len);
+		bytes -= len;
+	} while (bytes);
+
+	if (WARN_ON_ONCE(i != segs))
+		return -EIO;
+	return 0;
+}
+
 static int crypt_convert_block_skcipher(struct crypt_config *cc,
 					struct convert_context *ctx,
 					struct skcipher_request *req,
 					unsigned int tag_offset)
 {
-	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
-	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
 	struct scatterlist *sg_in, *sg_out;
 	struct dm_crypt_request *dmreq;
 	u8 *iv, *org_iv, *tag_iv;
 	__le64 *sector;
-	int r = 0;
-
-	/* Reject unexpected unaligned bio. */
-	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
-		return -EIO;
+	int r;
 
 	dmreq = dmreq_of_req(cc, req);
 	dmreq->iv_sector = ctx->cc_sector;
@@ -1456,15 +1484,18 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	sector = org_sector_of_dmreq(cc, dmreq);
 	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
 
-	/* For skcipher we use only the first sg item */
 	sg_in  = &dmreq->sg_in[0];
 	sg_out = &dmreq->sg_out[0];
 
-	sg_init_table(sg_in, 1);
-	sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
+	r = crypt_build_sgl(cc, sg_in, &ctx->iter_in, ctx->bio_in,
+			    ARRAY_SIZE(dmreq->sg_in));
+	if (r < 0)
+		return r;
 
-	sg_init_table(sg_out, 1);
-	sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
+	r = crypt_build_sgl(cc, sg_out, &ctx->iter_out, ctx->bio_out,
+			    ARRAY_SIZE(dmreq->sg_out));
+	if (r < 0)
+		return r;
 
 	if (cc->iv_gen_ops) {
 		/* For READs use IV stored in integrity metadata */
@@ -1495,9 +1526,6 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
 		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
 
-	bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
-	bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
-
 	return r;
 }
 
@@ -2829,10 +2857,12 @@ static void crypt_dtr(struct dm_target *ti)
 static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 {
 	struct crypt_config *cc = ti->private;
+	bool unaligned_allowed = true;
 
-	if (crypt_integrity_aead(cc))
+	if (crypt_integrity_aead(cc)) {
 		cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
-	else
+		unaligned_allowed = false;
+	} else
 		cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
 
 	if (cc->iv_size)
@@ -2868,6 +2898,7 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 		if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
 			return -EINVAL;
 		set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);
+		unaligned_allowed = false;
 	} else if (strcmp(ivmode, "lmk") == 0) {
 		cc->iv_gen_ops = &crypt_iv_lmk_ops;
 		/*
@@ -2880,10 +2911,12 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 			cc->key_parts++;
 			cc->key_extra_size = cc->key_size / cc->key_parts;
 		}
+		unaligned_allowed = false;
 	} else if (strcmp(ivmode, "tcw") == 0) {
 		cc->iv_gen_ops = &crypt_iv_tcw_ops;
 		cc->key_parts += 2; /* IV + whitening */
 		cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
+		unaligned_allowed = false;
 	} else if (strcmp(ivmode, "random") == 0) {
 		cc->iv_gen_ops = &crypt_iv_random_ops;
 		/* Need storage space in integrity fields. */
@@ -2893,6 +2926,12 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 		return -EINVAL;
 	}
 
+	if (!unaligned_allowed) {
+		cc->io_alignment = cc->sector_size - 1;
+	} else {
+		set_bit(CRYPT_DISCONTIGUOUS_SEGS, &cc->cipher_flags);
+		cc->io_alignment = 3;
+	}
 	return 0;
 }
 
@@ -3750,7 +3789,11 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
 	limits->physical_block_size =
 		max_t(unsigned int, limits->physical_block_size, cc->sector_size);
 	limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);
-	limits->dma_alignment = limits->logical_block_size - 1;
+
+	if (test_bit(CRYPT_DISCONTIGUOUS_SEGS, &cc->cipher_flags))
+		limits->dma_alignment = cc->io_alignment;
+	else
+		limits->dma_alignment = limits->logical_block_size - 1;
 
 	/*
 	 * For zoned dm-crypt targets, there will be no internal splitting of
drivers/md/dm-table.c +1 -0
index ad0a60a07b935..da0d090675103 100644
@@ -1789,6 +1789,7 @@ int dm_calculate_queue_limits(struct dm_table *t,
 	bool zoned = false;
 
 	dm_set_stacking_limits(limits);
+	limits->dma_alignment = 0;
 
 	t->integrity_supported = true;
 	for (unsigned int i = 0; i < t->num_targets; i++) {
Signature · 1 line
2.47.3
#2 lore raw

PATCH RESEND v3 2/2 dm-crypt: dynamic scatterlist for many segments

In reply to Keith Busch

From: Keith Busch <[email protected]>

In the unlikely case where the base bio uses a highly fragmented vector, the four inline scatterlist elements may not be enough, so allocate a temporary scatterlist when needed.

---
 drivers/md/dm-crypt.c | 51 +++++++++++++++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 11 deletions(-)
drivers/md/dm-crypt.c +40 -11
index 5304954b4574b..d659ea4573192 100644
@@ -98,6 +98,8 @@ struct dm_crypt_request {
 	struct scatterlist sg_in[4];
 	struct scatterlist sg_out[4];
 	u64 iv_sector;
+	struct scatterlist *__sg_in;
+	struct scatterlist *__sg_out;
 };
 
 struct crypt_config;
@@ -1426,18 +1428,32 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
 	return r;
 }
 
-static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist *sg,
+static void crypt_free_sgls(struct dm_crypt_request *dmreq)
+{
+	if (dmreq->__sg_in != dmreq->sg_in)
+		kfree(dmreq->__sg_in);
+	if (dmreq->__sg_out != dmreq->sg_out)
+		kfree(dmreq->__sg_out);
+	dmreq->__sg_in = NULL;
+	dmreq->__sg_out = NULL;
+}
+
+static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist **psg,
 			   struct bvec_iter *iter, struct bio *bio,
 			   int max_segs)
 {
 	unsigned int bytes = cc->sector_size;
+	struct scatterlist *sg = *psg;
 	struct bvec_iter tmp = *iter;
 	int segs, i = 0;
 
 	bio_advance_iter(bio, &tmp, bytes);
 	segs = tmp.bi_idx - iter->bi_idx + !!tmp.bi_bvec_done;
-	if (segs > max_segs)
-		return -EIO;
+	if (segs > max_segs) {
+		sg = kmalloc_array(segs, sizeof(struct scatterlist), GFP_NOIO);
+		if (!sg)
+			return -ENOMEM;
+	}
 
 	sg_init_table(sg, segs);
 	do {
@@ -1446,7 +1462,7 @@ static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist *sg,
 
 		/* Reject unexpected unaligned bio. */
 		if (unlikely((len | bv.bv_offset) & cc->io_alignment))
-			return -EIO;
+			goto error;
 
 		sg_set_page(&sg[i++], bv.bv_page, len, bv.bv_offset);
 		bio_advance_iter_single(bio, iter, len);
@@ -1454,8 +1470,13 @@ static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist *sg,
 	} while (bytes);
 
 	if (WARN_ON_ONCE(i != segs))
-		return -EIO;
+		goto error;
+	*psg = sg;
 	return 0;
+error:
+	if (sg != *psg)
+		kfree(sg);
+	return -EIO;
 }
 
 static int crypt_convert_block_skcipher(struct crypt_config *cc,
@@ -1484,18 +1505,21 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	sector = org_sector_of_dmreq(cc, dmreq);
 	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
 
-	sg_in  = &dmreq->sg_in[0];
-	sg_out = &dmreq->sg_out[0];
+	dmreq->__sg_in = &dmreq->sg_in[0];
+	dmreq->__sg_out = &dmreq->sg_out[0];
 
-	r = crypt_build_sgl(cc, sg_in, &ctx->iter_in, ctx->bio_in,
+	r = crypt_build_sgl(cc, &dmreq->__sg_in, &ctx->iter_in, ctx->bio_in,
 			    ARRAY_SIZE(dmreq->sg_in));
 	if (r < 0)
 		return r;
 
-	r = crypt_build_sgl(cc, sg_out, &ctx->iter_out, ctx->bio_out,
+	r = crypt_build_sgl(cc, &dmreq->__sg_out, &ctx->iter_out, ctx->bio_out,
 			    ARRAY_SIZE(dmreq->sg_out));
 	if (r < 0)
-		return r;
+		goto out;
+
+	sg_in = dmreq->__sg_in;
+	sg_out = dmreq->__sg_out;
 
 	if (cc->iv_gen_ops) {
 		/* For READs use IV stored in integrity metadata */
@@ -1504,7 +1528,7 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 		} else {
 			r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
 			if (r < 0)
-				return r;
+				goto out;
 			/* Data can be already preprocessed in generator */
 			if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
 				sg_in = sg_out;
@@ -1525,6 +1549,9 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 
 	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
 		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
+out:
+	if (r != -EINPROGRESS && r != -EBUSY)
+		crypt_free_sgls(dmreq);
 
 	return r;
 }
@@ -1590,7 +1617,9 @@ static void crypt_free_req_skcipher(struct crypt_config *cc,
 				    struct skcipher_request *req, struct bio *base_bio)
 {
 	struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
+	struct dm_crypt_request *dmreq = dmreq_of_req(cc, req);
 
+	crypt_free_sgls(dmreq);
 	if ((struct skcipher_request *)(io + 1) != req)
 		mempool_free(req, &cc->req_pool);
 }
Signature · 1 line
2.47.3