dm-integrity asynchronous hash patches
Subsystems (2)
Mikulas Patocka
maintainerPATCH 7/7 dm-integrity: enable asynchronous hash interface
In reply to Mikulas PatockaThis commit enables the asynchronous hash interface in dm-integrity.
--- drivers/md/dm-integrity.c | 196 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 162 insertions(+), 34 deletions(-)
drivers/md/dm-integrity.c +162 -34
@@ -224,6 +224,7 @@ struct dm_integrity_c {int failed;struct crypto_shash *internal_shash; +struct crypto_ahash *internal_ahash;unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1636,8 +1637,8 @@ static void integrity_end_io(struct biodec_in_flight (dio); } -static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, -const char *data,unsigned offset,char *result) +static void integrity_sector_checksum_shash (struct dm_integrity_c *ic,sector_t sector, +const char *data,unsigned offset,char *result) {__le64 sector_le =cpu_to_le64 (sector);SHASH_DESC_ON_STACK (req, ic->internal_shash);
@@ -1689,14 +1690,84 @@ failed:get_random_bytes (result, ic->tag_size); } +static void integrity_sector_checksum_ahash (struct dm_integrity_c *ic,sector_t sector, +struct page *page,unsigned offset,char *result) +{ +__le64 sector_le =cpu_to_le64 (sector); +HASH_REQUEST_ON_STACK (req, ic->internal_ahash); +DECLARE_CRYPTO_WAIT (wait); +struct scatterlist sg[3 ], *s = sg; +int r; +unsigned int digest_size; +unsigned int nbytes =0 ; + +might_sleep (); + + req->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK ; +ahash_request_set_callback (req,0 , crypto_req_done, &wait); + +if (ic->sb->flags &cpu_to_le32 (SB_FLAG_FIXED_HMAC )) { +sg_init_table (sg,3 ); +sg_set_buf (s, (const __u8 *)&ic->sb->salt,SALT_SIZE ); + nbytes +=SALT_SIZE ; + s++; + }else { +sg_init_table (sg,2 ); + } + +if (likely (!is_vmalloc_addr (§or_le))) { +sg_set_buf (s, §or_le,sizeof (sector_le)); + }else { +struct page *sec_page =vmalloc_to_page (§or_le); +unsigned int sec_off =offset_in_page (§or_le); +sg_set_page (s, sec_page,sizeof (sector_le), sec_off); + } + s++; + nbytes +=sizeof (sector_le); + +sg_set_page (s, page, ic->sectors_per_block <<SECTOR_SHIFT , offset); + nbytes += ic->sectors_per_block <<SECTOR_SHIFT ; + +ahash_request_set_crypt (req, sg, result, nbytes); + + r =crypto_wait_req (crypto_ahash_digest (req), &wait); +if (unlikely (r)) { +dm_integrity_io_error (ic,"crypto_ahash_digest" , r); + goto failed; + } + + digest_size = ic->internal_hash_digestsize; +if (unlikely (digest_size < ic->tag_size)) +memset (result + digest_size,0 , ic->tag_size - digest_size); + +return ; + +failed: +/* this shouldn't happen anyway, the hash functions have no reason to fail */ +get_random_bytes (result, ic->tag_size); +} + +static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, +const char *data,unsigned offset,char *result) +{ +if (likely (ic->internal_shash !=NULL )) +integrity_sector_checksum_shash (ic, sector, data, offset, result); +else +integrity_sector_checksum_ahash (ic, sector, (struct page *)data, offset, result); +} +static void *integrity_kmap (struct dm_integrity_c *ic,struct page *p) { -return kmap_local_page (p); +if (likely (ic->internal_shash !=NULL )) +return kmap_local_page (p); +else +return p; }static void integrity_kunmap (struct dm_integrity_c *ic,const void *ptr) { -kunmap_local (ptr); +if (likely (ic->internal_shash !=NULL )) +kunmap_local (ptr); }static void *integrity_identity (struct dm_integrity_c *ic,void *data)
@@ -1705,7 +1776,10 @@ static void *integrity_identity(struct dBUG_ON (offset_in_page (data));BUG_ON (!virt_addr_valid (data));#endif -return data; +if (likely (ic->internal_shash !=NULL )) +return data; +else +return virt_to_page (data); } static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
@@ -2623,32 +2697,58 @@ static void dm_integrity_inline_recheck(bio_endio (bio); } +static inline bool dm_integrity_check (struct dm_integrity_c *ic,struct dm_integrity_io *dio) +{ +struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io )); +unsigned pos =0 ; + +while (dio->bio_details.bi_iter.bi_size) { +char digest[HASH_MAX_DIGESTSIZE ]; +struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); +char *mem =integrity_kmap (ic, bv.bv_page); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); +if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, +min (ic->internal_hash_digestsize, ic->tag_size)))) { +integrity_kunmap (ic, mem); +dm_integrity_free_payload (dio); +INIT_WORK (&dio->work, dm_integrity_inline_recheck); +queue_work (ic->offload_wq, &dio->work); +return false; + } +integrity_kunmap (ic, mem); + pos += ic->tuple_size; +bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); + } + +return true; +} + +static void dm_integrity_inline_async_check (struct work_struct *w) +{ + struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); +struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io )); +struct dm_integrity_c *ic = dio->ic; + +if (likely (dm_integrity_check (ic, dio))) +bio_endio (bio); +} +static int dm_integrity_end_io (struct dm_target *ti,struct bio *bio,blk_status_t *status) {struct dm_integrity_c *ic = ti->private;if (ic->mode =='I' ) {struct dm_integrity_io *dio =dm_per_bio_data (bio,sizeof (struct dm_integrity_io )); -if (dio->op ==REQ_OP_READ &&likely (*status ==BLK_STS_OK )) { -unsigned pos =0 ; +if (dio->op ==REQ_OP_READ &&likely (*status ==BLK_STS_OK ) &&likely (dio->bio_details.bi_iter.bi_size !=0 )) {if (ic->sb->flags &cpu_to_le32 (SB_FLAG_RECALCULATING ) &&unlikely (dio->integrity_range_locked)) - goto skip_check; -while (dio->bio_details.bi_iter.bi_size) { -char digest[HASH_MAX_DIGESTSIZE ]; -struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =integrity_kmap (ic, bv.bv_page); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); -if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, -min (ic->internal_hash_digestsize, ic->tag_size)))) { -integrity_kunmap (ic, mem); -dm_integrity_free_payload (dio); -INIT_WORK (&dio->work, dm_integrity_inline_recheck); -queue_work (ic->offload_wq, &dio->work); + goto skip_check; +if (likely (ic->internal_shash !=NULL )) { +if (unlikely (!dm_integrity_check (ic, dio)))return DM_ENDIO_INCOMPLETE ; - } -integrity_kunmap (ic, mem); - pos += ic->tuple_size; -bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); + }else { +INIT_WORK (&dio->work, dm_integrity_inline_async_check); +queue_work (ic->offload_wq, &dio->work); +return DM_ENDIO_INCOMPLETE ; } } skip_check:
@@ -4229,27 +4329,49 @@ nomem:return -ENOMEM ; } -static int get_mac (struct crypto_shash **hash,struct alg_spec *a,char **error, -char *error_alg,char *error_key) +static int get_mac (struct crypto_shash **shash,struct crypto_ahash **ahash, +struct alg_spec *a,char **error,char *error_alg,char *error_key) {int r;if (a->alg_string) { - *hash =crypto_alloc_shash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); -if (IS_ERR (*hash)) { +if (ahash) { + *ahash =crypto_alloc_ahash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); +if (IS_ERR (*ahash)) { + *ahash =NULL ; + goto try_shash; + } + +if (a->key) { + r =crypto_ahash_setkey (*ahash, a->key, a->key_size); +if (r) { + *error = error_key; +return r; + } + }else if (crypto_ahash_get_flags (*ahash) &CRYPTO_TFM_NEED_KEY ) { + *error = error_key; +return -ENOKEY ; + } + +return 0 ; + } + +try_shash: + *shash =crypto_alloc_shash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); +if (IS_ERR (*shash)) { *error = error_alg; - r =PTR_ERR (*hash); - *hash =NULL ; + r =PTR_ERR (*shash); + *shash =NULL ;return r; }if (a->key) { - r =crypto_shash_setkey (*hash, a->key, a->key_size); + r =crypto_shash_setkey (*shash, a->key, a->key_size);if (r) { *error = error_key;return r; } - }else if (crypto_shash_get_flags (*hash) &CRYPTO_TFM_NEED_KEY ) { + }else if (crypto_shash_get_flags (*shash) &CRYPTO_TFM_NEED_KEY ) { *error = error_key;return -ENOKEY ; }
@@ -4709,7 +4831,7 @@ static int dm_integrity_ctr(struct dm_ta buffer_sectors =1 ; ic->log2_buffer_sectors =min ((int )__fls (buffer_sectors),31 -SECTOR_SHIFT ); - r =get_mac (&ic->internal_shash, &ic->internal_hash_alg, &ti->error, + r =get_mac (&ic->internal_shash, &ic->internal_ahash, &ic->internal_hash_alg, &ti->error,"Invalid internal hash" ,"Error setting internal hash key" );if (r) goto bad;
@@ -4717,8 +4839,12 @@ static int dm_integrity_ctr(struct dm_ta ic->internal_hash = true; ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_shash); } +if (ic->internal_ahash) { + ic->internal_hash = true; + ic->internal_hash_digestsize =crypto_ahash_digestsize (ic->internal_ahash); + } - r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error, + r =get_mac (&ic->journal_mac,NULL , &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );if (r) goto bad;
@@ -5240,6 +5366,8 @@ static void dm_integrity_dtr(struct dm_tif (ic->internal_shash)crypto_free_shash (ic->internal_shash); +if (ic->internal_ahash) +crypto_free_ahash (ic->internal_ahash);free_alg (&ic->internal_hash_alg);if (ic->journal_crypt)
@@ -5256,7 +5384,7 @@ static void dm_integrity_dtr(struct dm_tstatic struct target_type integrity_target = { .name ="integrity" , - .version = {1 ,13 ,0 }, + .version = {1 ,14 ,0 }, .module =THIS_MODULE , .features =DM_TARGET_SINGLETON |DM_TARGET_INTEGRITY , .ctr = dm_integrity_ctr,
Mikulas Patocka
maintainerPATCH 2/7 dm-integrity: replace bvec_kmap_local with kmap_local_page
In reply to Mikulas PatockaReplace bvec_kmap_local with kmap_local_page - it will be needed for the upcoming patches that make kmap_local_page optional, depending on whether asynchronous hash interface is used or not.
--- drivers/md/dm-integrity.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
drivers/md/dm-integrity.c +6 -7
@@ -1838,11 +1838,11 @@ static void integrity_metadata(struct wochar *mem, *checksums_ptr; again: - mem =bvec_kmap_local (&bv_copy); + mem =kmap_local_page (bv_copy.bv_page); pos =0 ; checksums_ptr = checksums; do { - integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); + integrity_sector_checksum(ic, sector, mem + bv_copy.bv_offset + pos, checksums_ptr); checksums_ptr += ic->tag_size; sectors_to_process -= ic->sectors_per_block; pos += ic->sectors_per_block << SECTOR_SHIFT;
@@ -2506,10 +2506,10 @@ skip_spinlock:unsigned pos =0 ;while (dio->bio_details.bi_iter.bi_size) {struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -const char *mem =bvec_kmap_local (&bv); +const char *mem =kmap_local_page (bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos);kunmap_local (mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT );
@@ -2626,9 +2626,8 @@ static int dm_integrity_end_io(struct dmwhile (dio->bio_details.bi_iter.bi_size) {char digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =bvec_kmap_local (&bv); -//memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, digest); +char *mem =kmap_local_page (bv.bv_page); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) {kunmap_local (mem);
Mikulas Patocka
maintainerPATCH 3/7 dm-integrity: introduce integrity_kmap and integrity_kunmap
In reply to Mikulas PatockaThis abstraction will be used later, for the asynchronous hash interface.
--- drivers/md/dm-integrity.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-)
drivers/md/dm-integrity.c +17 -20
@@ -1688,6 +1688,16 @@ failed:get_random_bytes (result, ic->tag_size); } +static void *integrity_kmap (struct dm_integrity_c *ic,struct page *p) +{ +return kmap_local_page (p); +} + +static void integrity_kunmap (struct dm_integrity_c *ic,const void *ptr) +{ +kunmap_local (ptr); +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) {struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));
@@ -1838,7 +1848,7 @@ static void integrity_metadata(struct wochar *mem, *checksums_ptr; again: - mem =kmap_local_page (bv_copy.bv_page); + mem =integrity_kmap (ic, bv_copy.bv_page); pos =0 ; checksums_ptr = checksums; do {
@@ -1848,7 +1858,7 @@ again: pos += ic->sectors_per_block <<SECTOR_SHIFT ; sector += ic->sectors_per_block; }while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack); -kunmap_local (mem); +integrity_kunmap (ic, mem); r =dm_integrity_rw_tag (ic, checksums, &dio->metadata_block, &dio->metadata_offset, checksums_ptr - checksums, dio->op ==REQ_OP_READ ?TAG_CMP :TAG_WRITE );
@@ -2072,19 +2082,6 @@ retry_kmap: js++; mem_ptr +=1 <<SECTOR_SHIFT ; }while (++s < ic->sectors_per_block); -#ifdef INTERNAL_VERIFY -if (ic->internal_hash) { -char checksums_onstack[MAX_T (size_t,HASH_MAX_DIGESTSIZE ,MAX_TAG_SIZE )]; - -integrity_sector_checksum (ic, logical_sector, mem + bv.bv_offset, checksums_onstack); -if (unlikely (crypto_memneq (checksums_onstack,journal_entry_tag (ic, je), ic->tag_size))) { -DMERR_LIMIT ("Checksum failed when reading from journal, at sector 0x%llx" , - logical_sector); -dm_audit_log_bio (DM_MSG_PREFIX ,"journal-checksum" , - bio, logical_sector,0 ); - } - } -#endif }if (!ic->internal_hash) {
@@ -2506,11 +2503,11 @@ skip_spinlock:unsigned pos =0 ;while (dio->bio_details.bi_iter.bi_size) {struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -const char *mem =kmap_local_page (bv.bv_page); +const char *mem =integrity_kmap (ic, bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos); -kunmap_local (mem); +integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); }
@@ -2626,17 +2623,17 @@ static int dm_integrity_end_io(struct dmwhile (dio->bio_details.bi_iter.bi_size) {char digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =kmap_local_page (bv.bv_page); +char *mem =integrity_kmap (ic, bv.bv_page);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) { -kunmap_local (mem); +integrity_kunmap (ic, mem);dm_integrity_free_payload (dio);INIT_WORK (&dio->work, dm_integrity_inline_recheck);queue_work (ic->offload_wq, &dio->work);return DM_ENDIO_INCOMPLETE ; } -kunmap_local (mem); +integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); }
Mikulas Patocka
maintainercover dm-integrity asynchronous hash patches
Hi
Here I'm sending the first version of the new dm-integrity asynchronous hash patches.
The patches prefer the asynchronous hash interface over the synchronous hash interface - it is for testing and it will be changed in the final release.
Mikulas
Mikulas Patocka
maintainerPATCH 4/7 dm-integrity: allocate the recalculate buffer with kmalloc
Allocate the recalculate buffer with kmalloc rather than vmalloc. This will be needed later, for the simplification of the asynchronous hash interface.
--- drivers/md/dm-integrity.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/md/dm-integrity.c +4 -4
@@ -2998,7 +2998,7 @@ static void integrity_recalc(struct workunsigned recalc_sectors =RECALC_SECTORS ; retry: - recalc_buffer =__vmalloc (recalc_sectors <<SECTOR_SHIFT ,GFP_NOIO ); + recalc_buffer =kmalloc (recalc_sectors <<SECTOR_SHIFT ,GFP_NOIO | __GFP_NOWARN);if (!recalc_buffer) { oom: recalc_sectors >>=1 ;
@@ -3012,7 +3012,7 @@ oom: recalc_tags_size += ic->internal_hash_digestsize - ic->tag_size; recalc_tags =kvmalloc (recalc_tags_size,GFP_NOIO );if (!recalc_tags) { -vfree (recalc_buffer); +kfree (recalc_buffer); recalc_buffer =NULL ; goto oom; }
@@ -3078,7 +3078,7 @@ next_chunk: goto err; io_req.bi_opf =REQ_OP_READ ; - io_req.mem.type =DM_IO_VMA ; + io_req.mem.type =DM_IO_KMEM ; io_req.mem.ptr.addr = recalc_buffer; io_req.notify.fn =NULL ; io_req.client = ic->io;
@@ -3136,7 +3136,7 @@ unlock_ret:recalc_write_super (ic); free_ret: -vfree (recalc_buffer); +kfree (recalc_buffer);kvfree (recalc_tags); }
Mikulas Patocka
maintainerPATCH 1/7 dm-integrity: use internal variable for digestsize
In reply to Mikulas PatockaInstead of calling digestsize() each time the digestsize for the internal hash is needed, store the digestsize in a new field internal_hash_digestsize within struct dm_integrity_c once and use this value when needed.
--- drivers/md/dm-integrity.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-)
drivers/md/dm-integrity.c +14 -11
@@ -223,6 +223,7 @@ struct dm_integrity_c {int failed;struct crypto_shash *internal_hash; +unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1676,7 +1677,7 @@ static void integrity_sector_checksum(st goto failed; } - digest_size =crypto_shash_digestsize (ic->internal_hash); + digest_size = ic->internal_hash_digestsize;if (unlikely (digest_size < ic->tag_size))memset (result + digest_size,0 , ic->tag_size - digest_size);
@@ -1776,7 +1777,7 @@ static void integrity_metadata(struct woif (ic->internal_hash) {struct bvec_iter iter;struct bio_vec bv; -unsigned int digest_size =crypto_shash_digestsize (ic->internal_hash); +unsigned int digest_size = ic->internal_hash_digestsize;struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));char *checksums;unsigned int extra_space =unlikely (digest_size > ic->tag_size) ? digest_size - ic->tag_size :0 ;
@@ -2124,7 +2125,7 @@ retry_kmap: }while (++s < ic->sectors_per_block);if (ic->internal_hash) { -unsigned int digest_size =crypto_shash_digestsize (ic->internal_hash); +unsigned int digest_size = ic->internal_hash_digestsize;if (unlikely (digest_size > ic->tag_size)) {char checksums_onstack[HASH_MAX_DIGESTSIZE ];
@@ -2428,7 +2429,7 @@ retry:if (!dio->integrity_payload) {unsigned digest_size, extra_size; dio->payload_len = ic->tuple_size * (bio_sectors (bio) >> ic->sb->log2_sectors_per_block); - digest_size =crypto_shash_digestsize (ic->internal_hash); + digest_size = ic->internal_hash_digestsize; extra_size =unlikely (digest_size > ic->tag_size) ? digest_size - ic->tag_size :0 ; dio->payload_len += extra_size; dio->integrity_payload =kmalloc (dio->payload_len,GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
@@ -2589,7 +2590,7 @@ static void dm_integrity_inline_recheck(bio_put (outgoing_bio);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest); -if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (crypto_shash_digestsize (ic->internal_hash), ic->tag_size)))) { +if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (ic->internal_hash_digestsize, ic->tag_size)))) {DMERR_LIMIT ("%pg: Checksum failed at sector 0x%llx" , ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);atomic64_inc (&ic->number_of_mismatches);
@@ -2629,7 +2630,7 @@ static int dm_integrity_end_io(struct dm//memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT); integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, -min (crypto_shash_digestsize (ic->internal_hash), ic->tag_size)))) { +min (ic->internal_hash_digestsize, ic->tag_size)))) {kunmap_local (mem);dm_integrity_free_payload (dio);INIT_WORK (&dio->work, dm_integrity_inline_recheck);
@@ -3011,8 +3012,8 @@ oom: goto free_ret; } recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; -if (crypto_shash_digestsize (ic->internal_hash) > ic->tag_size) - recalc_tags_size +=crypto_shash_digestsize (ic->internal_hash) - ic->tag_size; +if (ic->internal_hash_digestsize > ic->tag_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tag_size; recalc_tags =kvmalloc (recalc_tags_size,GFP_NOIO );if (!recalc_tags) {vfree (recalc_buffer);
@@ -3171,8 +3172,8 @@ oom: } recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size; -if (crypto_shash_digestsize (ic->internal_hash) > ic->tuple_size) - recalc_tags_size +=crypto_shash_digestsize (ic->internal_hash) - ic->tuple_size; +if (ic->internal_hash_digestsize > ic->tuple_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tuple_size; recalc_tags =kmalloc (recalc_tags_size,GFP_NOIO | __GFP_NOWARN);if (!recalc_tags) {kfree (recalc_buffer);
@@ -4694,6 +4695,8 @@ static int dm_integrity_ctr(struct dm_ta "Invalid internal hash", "Error setting internal hash key");if (r) goto bad; +if (ic->internal_hash) + ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_hash); r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );
@@ -4706,7 +4709,7 @@ static int dm_integrity_ctr(struct dm_ta r = -EINVAL ; goto bad; } - ic->tag_size =crypto_shash_digestsize (ic->internal_hash); + ic->tag_size = ic->internal_hash_digestsize; }if (ic->tag_size >MAX_TAG_SIZE ) { ti->error ="Too big tag size" ;
Mikulas Patocka
maintainerPATCH 5/7 dm-integrity: add the "offset" argument
In reply to Mikulas PatockaMake sure that the "data" argument passed to integrity_sector_checksum is always page-aligned and add an "offset" argument that specifies the offset from the start of the page. This will enable us to use the asynchronous hash interface later.
--- drivers/md/dm-integrity.c | 49 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-)
drivers/md/dm-integrity.c +35 -14
@@ -1636,7 +1636,7 @@ static void integrity_end_io(struct bio }static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, -const char *data,char *result) +const char *data,unsigned offset,char *result) {__le64 sector_le =cpu_to_le64 (sector);SHASH_DESC_ON_STACK (req, ic->internal_hash);
@@ -1665,7 +1665,7 @@ static void integrity_sector_checksum(st goto failed; } - r =crypto_shash_update (req, data, ic->sectors_per_block <<SECTOR_SHIFT ); + r =crypto_shash_update (req, data + offset, ic->sectors_per_block <<SECTOR_SHIFT );if (unlikely (r <0 )) {dm_integrity_io_error (ic,"crypto_shash_update" , r); goto failed;
@@ -1698,6 +1698,15 @@ static void integrity_kunmap(struct dm_ikunmap_local (ptr); } +static void *integrity_identity (struct dm_integrity_c *ic,void *data) +{ +#ifdef CONFIG_DEBUG_SG +BUG_ON (offset_in_page (data)); +BUG_ON (!virt_addr_valid (data)); +#endif +return data; +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) {struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));
@@ -1722,6 +1731,7 @@ static noinline void integrity_recheck(ssector_t alignment;char *mem;char *buffer =page_to_virt (page); +unsigned int buffer_offset;int r;struct dm_io_request io_req;struct dm_io_region io_loc;
@@ -1739,7 +1749,7 @@ static noinline void integrity_recheck(s alignment &= -alignment; io_loc.sector =round_down (io_loc.sector, alignment); io_loc.count += sector - io_loc.sector; - buffer += (sector - io_loc.sector) <<SECTOR_SHIFT ; + buffer_offset = (sector - io_loc.sector) <<SECTOR_SHIFT ; io_loc.count =round_up (io_loc.count, alignment); r =dm_io (&io_req,1 , &io_loc,NULL ,IOPRIO_DEFAULT );
@@ -1748,7 +1758,7 @@ static noinline void integrity_recheck(s goto free_ret; } -integrity_sector_checksum (ic, logical_sector, buffer, checksum); +integrity_sector_checksum (ic, logical_sector,integrity_identity (ic, buffer), buffer_offset, checksum); r =dm_integrity_rw_tag (ic, checksum, &dio->metadata_block, &dio->metadata_offset, ic->tag_size,TAG_CMP );if (r) {
@@ -1765,7 +1775,7 @@ static noinline void integrity_recheck(s } mem =bvec_kmap_local (&bv); -memcpy (mem + pos, buffer, ic->sectors_per_block <<SECTOR_SHIFT ); +memcpy (mem + pos, buffer + buffer_offset, ic->sectors_per_block <<SECTOR_SHIFT );kunmap_local (mem); pos += ic->sectors_per_block <<SECTOR_SHIFT ;
@@ -1852,7 +1862,7 @@ again: pos =0 ; checksums_ptr = checksums; do { - integrity_sector_checksum(ic, sector, mem + bv_copy.bv_offset + pos, checksums_ptr); + integrity_sector_checksum(ic, sector, mem, bv_copy.bv_offset + pos, checksums_ptr); checksums_ptr += ic->tag_size; sectors_to_process -= ic->sectors_per_block; pos += ic->sectors_per_block << SECTOR_SHIFT;
@@ -2123,14 +2133,16 @@ retry_kmap:if (ic->internal_hash) {unsigned int digest_size = ic->internal_hash_digestsize; +void *js_page =integrity_identity (ic, (char *)js -offset_in_page (js)); +unsigned js_offset =offset_in_page (js);if (unlikely (digest_size > ic->tag_size)) {char checksums_onstack[HASH_MAX_DIGESTSIZE ]; -integrity_sector_checksum (ic, logical_sector, (char *)js, checksums_onstack); +integrity_sector_checksum (ic, logical_sector, js_page, js_offset, checksums_onstack);memcpy (journal_entry_tag (ic, je), checksums_onstack, ic->tag_size); }else -integrity_sector_checksum (ic, logical_sector, (char *)js,journal_entry_tag (ic, je)); +integrity_sector_checksum (ic, logical_sector, js_page, js_offset,journal_entry_tag (ic, je)); }journal_entry_set_sector (je, logical_sector);
@@ -2506,7 +2518,7 @@ skip_spinlock:const char *mem =integrity_kmap (ic, bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos);integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT );
@@ -2586,7 +2598,7 @@ static void dm_integrity_inline_recheck( }bio_put (outgoing_bio); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector,integrity_identity (ic, outgoing_data),0 , digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (ic->internal_hash_digestsize, ic->tag_size)))) {DMERR_LIMIT ("%pg: Checksum failed at sector 0x%llx" , ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);
@@ -2624,7 +2636,7 @@ static int dm_integrity_end_io(struct dmchar digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter);char *mem =integrity_kmap (ic, bv.bv_page); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) {integrity_kunmap (ic, mem);
@@ -2899,9 +2911,12 @@ static void do_journal_write(struct dm_i#endif ic->internal_hash) {char test_tag[MAX_T (size_t,HASH_MAX_DIGESTSIZE ,MAX_TAG_SIZE )]; +struct journal_sector *js =access_journal_data (ic, i, l); +void *js_page =integrity_identity (ic, (char *)js -offset_in_page (js)); +unsigned js_offset =offset_in_page (js);integrity_sector_checksum (ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), - (char *)access_journal_data (ic, i, l), test_tag); + js_page, js_offset, test_tag);if (unlikely (crypto_memneq (test_tag,journal_entry_tag (ic, je2), ic->tag_size))) {dm_integrity_io_error (ic,"tag mismatch when replaying journal" , -EILSEQ );dm_audit_log_target (DM_MSG_PREFIX ,"integrity-replay-journal" , ic->ti,0 );
@@ -3094,7 +3109,10 @@ next_chunk: t = recalc_tags;for (i =0 ; i < n_sectors; i += ic->sectors_per_block) { -integrity_sector_checksum (ic, logical_sector + i, recalc_buffer + (i <<SECTOR_SHIFT ), t); +void *ptr = recalc_buffer + (i <<SECTOR_SHIFT ); +void *ptr_page =integrity_identity (ic, (char *)ptr -offset_in_page (ptr)); +unsigned ptr_offset =offset_in_page (ptr); +integrity_sector_checksum (ic, logical_sector + i, ptr_page, ptr_offset, t); t += ic->tag_size; }
@@ -3214,8 +3232,11 @@ next_chunk: t = recalc_tags;for (i =0 ; i < range.n_sectors; i += ic->sectors_per_block) { +void *ptr = recalc_buffer + (i <<SECTOR_SHIFT ); +void *ptr_page =integrity_identity (ic, (char *)ptr -offset_in_page (ptr)); +unsigned ptr_offset =offset_in_page (ptr);memset (t,0 , ic->tuple_size); -integrity_sector_checksum (ic, range.logical_sector + i, recalc_buffer + (i <<SECTOR_SHIFT ), t); +integrity_sector_checksum (ic, range.logical_sector + i, ptr_page, ptr_offset, t); t += ic->tuple_size; }
Mikulas Patocka
maintainerPATCH 6/7 dm-integrity: rename internal_hash
In reply to Mikulas PatockaRename "internal_hash" to "internal_shash" and introduce a boolean value "internal_hash".
--- drivers/md/dm-integrity.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)
drivers/md/dm-integrity.c +11 -8
@@ -219,10 +219,11 @@ struct dm_integrity_c {__u8 log2_blocks_per_bitmap_bit;unsigned char mode; +bool internal_hash;int failed; -struct crypto_shash *internal_hash; +struct crypto_shash *internal_shash;unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1639,11 +1640,11 @@ static void integrity_sector_checksum(st const char *data, unsigned offset, char *result) { __le64 sector_le = cpu_to_le64(sector); -SHASH_DESC_ON_STACK (req, ic->internal_hash); +SHASH_DESC_ON_STACK (req, ic->internal_shash);int r;unsigned int digest_size; - req->tfm = ic->internal_hash; + req->tfm = ic->internal_shash; r =crypto_shash_init (req);if (unlikely (r <0 )) {
@@ -4708,12 +4709,14 @@ static int dm_integrity_ctr(struct dm_ta buffer_sectors =1 ; ic->log2_buffer_sectors =min ((int )__fls (buffer_sectors),31 -SECTOR_SHIFT ); - r =get_mac (&ic->internal_hash, &ic->internal_hash_alg, &ti->error, + r =get_mac (&ic->internal_shash, &ic->internal_hash_alg, &ti->error,"Invalid internal hash" ,"Error setting internal hash key" );if (r) goto bad; -if (ic->internal_hash) - ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_hash); +if (ic->internal_shash) { + ic->internal_hash = true; + ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_shash); + } r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );
@@ -5235,8 +5238,8 @@ static void dm_integrity_dtr(struct dm_tif (ic->sb)free_pages_exact (ic->sb,SB_SECTORS <<SECTOR_SHIFT ); -if (ic->internal_hash) -crypto_free_shash (ic->internal_hash); +if (ic->internal_shash) +crypto_free_shash (ic->internal_shash);free_alg (&ic->internal_hash_alg);if (ic->journal_crypt)