dm crypt: Allow unaligned buffer lengths for skcipher devices

cover

 drivers/md/dm-crypt.c | 154 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 108 insertions(+), 46 deletions(-)
1.8.3.1
---
 drivers/md/dm-crypt.c | 50 ++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 10 deletions(-)
index 380386c36921..9c26ad08732f 100644
@@ -1275,6 +1275,23 @@ static void *iv_tag_from_dmreq(struct crypt_config *cc,
 	return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
 }
 
+static int build_split_sg(struct scatterlist *sg, struct bio_vec *bvec,
+			  struct bio *bio, struct bvec_iter *iter,
+			  unsigned short int sector_size)
+{
+	int bytes_first_page;
+
+	bytes_first_page = bvec->bv_len;
+	sg_set_page(sg, bvec->bv_page, bvec->bv_len, bvec->bv_offset);
+	bio_advance_iter(bio, iter, bvec->bv_len);
+	*bvec = bio_iter_iovec(bio, *iter);
+	sg++;
+	sg_set_page(sg, bvec->bv_page, sector_size - bytes_first_page,
+		    bvec->bv_offset);
+
+	return bytes_first_page;
+}
+
 static int crypt_convert_block_aead(struct crypt_config *cc,
 				     struct convert_context *ctx,
 				     struct aead_request *req,

@@ -1379,15 +1396,12 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
 	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
 	struct scatterlist *sg_in, *sg_out;
+	int src_split = 0, dst_split = 0;
 	struct dm_crypt_request *dmreq;
 	u8 *iv, *org_iv, *tag_iv;
 	__le64 *sector;
 	int r = 0;
 
-	/* Reject unexpected unaligned bio. */
-	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
-		return -EIO;
-
 	dmreq = dmreq_of_req(cc, req);
 	dmreq->iv_sector = ctx->cc_sector;
 	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))

@@ -1407,11 +1421,25 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	sg_in  = &dmreq->sg_in[0];
 	sg_out = &dmreq->sg_out[0];
 
-	sg_init_table(sg_in, 1);
-	sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
+	if (unlikely(bv_in.bv_len < cc->sector_size)) {
+		sg_init_table(sg_in, 2);
+		src_split = build_split_sg(sg_in, &bv_in, ctx->bio_in,
+					   &ctx->iter_in, cc->sector_size);
+	} else {
+		sg_init_table(sg_in, 1);
+		sg_set_page(sg_in, bv_in.bv_page, cc->sector_size,
+			    bv_in.bv_offset);
+	}
 
-	sg_init_table(sg_out, 1);
-	sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
+	if (unlikely(bv_out.bv_len < cc->sector_size)) {
+		sg_init_table(sg_out, 2);
+		dst_split = build_split_sg(sg_out, &bv_out, ctx->bio_out,
+					   &ctx->iter_out, cc->sector_size);
+	} else {
+		sg_init_table(sg_out, 1);
+		sg_set_page(sg_out, bv_out.bv_page, cc->sector_size,
+			    bv_out.bv_offset);
+	}
 
 	if (cc->iv_gen_ops) {
 		/* For READs use IV stored in integrity metadata */

@@ -1442,8 +1470,10 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
 		r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
 
-	bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
-	bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
+	bio_advance_iter(ctx->bio_in, &ctx->iter_in,
+			 cc->sector_size - src_split);
+	bio_advance_iter(ctx->bio_out, &ctx->iter_out,
+			 cc->sector_size - dst_split);
 
 	return r;
 }

1.8.3.1
---
 drivers/md/dm-crypt.c | 104 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 68 insertions(+), 36 deletions(-)
index 9c26ad08732f..c40ada41d8ef 100644
@@ -471,11 +471,13 @@ static int crypt_iv_lmk_wipe(struct crypt_config *cc)
 
 static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
 			    struct dm_crypt_request *dmreq,
-			    u8 *data)
+			    struct scatterlist *sg)
 {
 	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
 	SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
+	struct sg_mapping_iter miter;
 	struct md5_state md5state;
+	size_t len = 16 * 31;
 	__le32 buf[4];
 	int i, r;
 

@@ -492,7 +494,19 @@ static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
 	}
 
 	/* Sector is always 512B, block size 16, add data of blocks 1-31 */
-	r = crypto_shash_update(desc, data + 16, 16 * 31);
+	sg_miter_start(&miter, sg, sg_nents(sg),
+		       SG_MITER_ATOMIC | SG_MITER_FROM_SG);
+	sg_miter_skip(&miter, 16);
+	while (sg_miter_next(&miter) && len > 0) {
+		size_t hash_len = min_t(size_t, miter.length, len);
+
+		r = crypto_shash_update(desc, miter.addr, hash_len);
+		if (r)
+			break;
+		len -= hash_len;
+	}
+	sg_miter_stop(&miter);
+
 	if (r)
 		return r;
 

@@ -520,15 +534,11 @@ static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
 static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
 			    struct dm_crypt_request *dmreq)
 {
-	struct scatterlist *sg;
-	u8 *src;
 	int r = 0;
 
 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
-		sg = crypt_get_sg_data(cc, dmreq->sg_in);
-		src = kmap_atomic(sg_page(sg));
-		r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);
-		kunmap_atomic(src);
+		r = crypt_iv_lmk_one(cc, iv, dmreq,
+				     crypt_get_sg_data(cc, dmreq->sg_in));
 	} else
 		memset(iv, 0, cc->iv_size);
 

@@ -538,22 +548,32 @@ static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
 static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
 			     struct dm_crypt_request *dmreq)
 {
+	struct sg_mapping_iter miter;
 	struct scatterlist *sg;
-	u8 *dst;
-	int r;
+	int r, offset = 0;
+	size_t len;
 
 	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
 		return 0;
 
 	sg = crypt_get_sg_data(cc, dmreq->sg_out);
-	dst = kmap_atomic(sg_page(sg));
-	r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);
+	r = crypt_iv_lmk_one(cc, iv, dmreq, sg);
+	if (r)
+		return r;
 
 	/* Tweak the first block of plaintext sector */
-	if (!r)
-		crypto_xor(dst + sg->offset, iv, cc->iv_size);
+	len = cc->iv_size;
+	sg_miter_start(&miter, sg, sg_nents(sg),
+		       SG_MITER_ATOMIC | SG_MITER_TO_SG);
+	while (sg_miter_next(&miter) && len > 0) {
+		size_t xor_len = min_t(size_t, miter.length, len);
+
+		crypto_xor(miter.addr, iv + offset, xor_len);
+		len -= xor_len;
+		offset += xor_len;
+	}
+	sg_miter_stop(&miter);
 
-	kunmap_atomic(dst);
 	return r;
 }
 

@@ -627,12 +647,14 @@ static int crypt_iv_tcw_wipe(struct crypt_config *cc)
 
 static int crypt_iv_tcw_whitening(struct crypt_config *cc,
 				  struct dm_crypt_request *dmreq,
-				  u8 *data)
+				  struct scatterlist *sg)
 {
 	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
 	__le64 sector = cpu_to_le64(dmreq->iv_sector);
+	struct sg_mapping_iter miter;
 	u8 buf[TCW_WHITENING_SIZE];
 	SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
+	size_t remain, sgoffset = 0;
 	int i, r;
 
 	/* xor whitening with sector number */

@@ -656,8 +678,31 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
 	crypto_xor(&buf[4], &buf[8], 4);
 
 	/* apply whitening (8 bytes) to whole sector */
-	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
-		crypto_xor(data + i * 8, buf, 8);
+	sg_miter_start(&miter, sg, sg_nents(sg),
+		       SG_MITER_ATOMIC | SG_MITER_TO_SG);
+	sg_miter_next(&miter);
+	remain = miter.length;
+	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++) {
+		size_t len = 8, offset = 0;
+
+		while (len > 0) {
+			size_t xor_len = min_t(size_t, remain, len);
+
+			crypto_xor(miter.addr + sgoffset, buf + offset,
+				   xor_len);
+			len -= xor_len;
+			remain -= xor_len;
+			offset += xor_len;
+			sgoffset += xor_len;
+			if (remain == 0) {
+				sg_miter_next(&miter);
+				sgoffset = 0;
+				remain = miter.length;
+			}
+		}
+	}
+	sg_miter_stop(&miter);
+
 out:
 	memzero_explicit(buf, sizeof(buf));
 	return r;

@@ -666,19 +711,14 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
 static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
 			    struct dm_crypt_request *dmreq)
 {
-	struct scatterlist *sg;
 	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
 	__le64 sector = cpu_to_le64(dmreq->iv_sector);
-	u8 *src;
 	int r = 0;
 
 	/* Remove whitening from ciphertext */
-	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
-		sg = crypt_get_sg_data(cc, dmreq->sg_in);
-		src = kmap_atomic(sg_page(sg));
-		r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);
-		kunmap_atomic(src);
-	}
+	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
+		r = crypt_iv_tcw_whitening(cc, dmreq,
+					   crypt_get_sg_data(cc, dmreq->sg_in));
 
 	/* Calculate IV */
 	crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);

@@ -692,20 +732,12 @@ static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
 static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
 			     struct dm_crypt_request *dmreq)
 {
-	struct scatterlist *sg;
-	u8 *dst;
-	int r;
-
 	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
 		return 0;
 
 	/* Apply whitening on ciphertext */
-	sg = crypt_get_sg_data(cc, dmreq->sg_out);
-	dst = kmap_atomic(sg_page(sg));
-	r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);
-	kunmap_atomic(dst);
-
-	return r;
+	return crypt_iv_tcw_whitening(cc, dmreq,
+				      crypt_get_sg_data(cc, dmreq->sg_out));
 }
 
 static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,

1.8.3.1
 drivers/md/dm-crypt.c | 154 +++++++++++++++++++++++++++++++++++----
-----------
 1 file changed, 108 insertions(+), 46 deletions(-)
 drivers/md/dm-crypt.c | 154 +++++++++++++++++++++++++++++++++++----
-----------
 1 file changed, 108 insertions(+), 46 deletions(-)
Damien Le Moal
Western Digital Research
 drivers/md/dm-crypt.c | 154
 1 file changed, 108 insertions(+), 46 deletions(-)

dm crypt: Allow unaligned buffer lengths for skcipher devices

Sudhakar Panneerselvam

Sudhakar Panneerselvam

Sudhakar Panneerselvam

Sudhakar Panneerselvam

Mike Snitzer

Eric Biggers

Milan Broz

Sudhakar Panneerselvam

Sudhakar Panneerselvam

Mikulas Patocka

Sudhakar Panneerselvam

Mikulas Patocka

Sudhakar Panneerselvam

Mikulas Patocka

Sudhakar Panneerselvam

Damien Le Moal

Mike Snitzer

Mikulas Patocka

Sudhakar Panneerselvam

Mikulas Patocka

Sudhakar Panneerselvam