dm-integrity asynchronous hash patches
Subsystems (2)
Mikulas Patocka
maintainercover dm-integrity asynchronous hash patches
Hi
Here I'm sending the first version of the new dm-integrity asynchronous hash patches.
The patches prefer the asynchronous hash interface over the synchronous hash interface - it is for testing and it will be changed in the final release.
Mikulas
Mikulas Patocka
maintainerPATCH 1/7 dm-integrity: use internal variable for digestsize
Instead of calling digestsize() each time the digestsize for the internal hash is needed, store the digestsize in a new field internal_hash_digestsize within struct dm_integrity_c once and use this value when needed.
--- drivers/md/dm-integrity.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-)
drivers/md/dm-integrity.c +14 -11
@@ -223,6 +223,7 @@ struct dm_integrity_c {int failed;struct crypto_shash *internal_hash; +unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1676,7 +1677,7 @@ static void integrity_sector_checksum(st goto failed; } - digest_size =crypto_shash_digestsize (ic->internal_hash); + digest_size = ic->internal_hash_digestsize;if (unlikely (digest_size < ic->tag_size))memset (result + digest_size,0 , ic->tag_size - digest_size);
@@ -1776,7 +1777,7 @@ static void integrity_metadata(struct woif (ic->internal_hash) {struct bvec_iter iter;struct bio_vec bv; -unsigned int digest_size =crypto_shash_digestsize (ic->internal_hash); +unsigned int digest_size = ic->internal_hash_digestsize;struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));char *checksums;unsigned int extra_space =unlikely (digest_size > ic->tag_size) ? digest_size - ic->tag_size :0 ;
@@ -2124,7 +2125,7 @@ retry_kmap: }while (++s < ic->sectors_per_block);if (ic->internal_hash) { -unsigned int digest_size =crypto_shash_digestsize (ic->internal_hash); +unsigned int digest_size = ic->internal_hash_digestsize;if (unlikely (digest_size > ic->tag_size)) {char checksums_onstack[HASH_MAX_DIGESTSIZE ];
@@ -2428,7 +2429,7 @@ retry:if (!dio->integrity_payload) {unsigned digest_size, extra_size; dio->payload_len = ic->tuple_size * (bio_sectors (bio) >> ic->sb->log2_sectors_per_block); - digest_size =crypto_shash_digestsize (ic->internal_hash); + digest_size = ic->internal_hash_digestsize; extra_size =unlikely (digest_size > ic->tag_size) ? digest_size - ic->tag_size :0 ; dio->payload_len += extra_size; dio->integrity_payload =kmalloc (dio->payload_len,GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
@@ -2589,7 +2590,7 @@ static void dm_integrity_inline_recheck(bio_put (outgoing_bio);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest); -if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (crypto_shash_digestsize (ic->internal_hash), ic->tag_size)))) { +if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (ic->internal_hash_digestsize, ic->tag_size)))) {DMERR_LIMIT ("%pg: Checksum failed at sector 0x%llx" , ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);atomic64_inc (&ic->number_of_mismatches);
@@ -2629,7 +2630,7 @@ static int dm_integrity_end_io(struct dm//memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT); integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, -min (crypto_shash_digestsize (ic->internal_hash), ic->tag_size)))) { +min (ic->internal_hash_digestsize, ic->tag_size)))) {kunmap_local (mem);dm_integrity_free_payload (dio);INIT_WORK (&dio->work, dm_integrity_inline_recheck);
@@ -3011,8 +3012,8 @@ oom: goto free_ret; } recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; -if (crypto_shash_digestsize (ic->internal_hash) > ic->tag_size) - recalc_tags_size +=crypto_shash_digestsize (ic->internal_hash) - ic->tag_size; +if (ic->internal_hash_digestsize > ic->tag_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tag_size; recalc_tags =kvmalloc (recalc_tags_size,GFP_NOIO );if (!recalc_tags) {vfree (recalc_buffer);
@@ -3171,8 +3172,8 @@ oom: } recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size; -if (crypto_shash_digestsize (ic->internal_hash) > ic->tuple_size) - recalc_tags_size +=crypto_shash_digestsize (ic->internal_hash) - ic->tuple_size; +if (ic->internal_hash_digestsize > ic->tuple_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tuple_size; recalc_tags =kmalloc (recalc_tags_size,GFP_NOIO | __GFP_NOWARN);if (!recalc_tags) {kfree (recalc_buffer);
@@ -4694,6 +4695,8 @@ static int dm_integrity_ctr(struct dm_ta "Invalid internal hash", "Error setting internal hash key");if (r) goto bad; +if (ic->internal_hash) + ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_hash); r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );
@@ -4706,7 +4709,7 @@ static int dm_integrity_ctr(struct dm_ta r = -EINVAL ; goto bad; } - ic->tag_size =crypto_shash_digestsize (ic->internal_hash); + ic->tag_size = ic->internal_hash_digestsize; }if (ic->tag_size >MAX_TAG_SIZE ) { ti->error ="Too big tag size" ;
Mikulas Patocka
maintainerPATCH 2/7 dm-integrity: replace bvec_kmap_local with kmap_local_page
In reply to Mikulas PatockaReplace bvec_kmap_local with kmap_local_page - it will be needed for the upcoming patches that make kmap_local_page optional, depending on whether asynchronous hash interface is used or not.
--- drivers/md/dm-integrity.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-)
drivers/md/dm-integrity.c +6 -7
@@ -1838,11 +1838,11 @@ static void integrity_metadata(struct wochar *mem, *checksums_ptr; again: - mem =bvec_kmap_local (&bv_copy); + mem =kmap_local_page (bv_copy.bv_page); pos =0 ; checksums_ptr = checksums; do { - integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); + integrity_sector_checksum(ic, sector, mem + bv_copy.bv_offset + pos, checksums_ptr); checksums_ptr += ic->tag_size; sectors_to_process -= ic->sectors_per_block; pos += ic->sectors_per_block << SECTOR_SHIFT;
@@ -2506,10 +2506,10 @@ skip_spinlock:unsigned pos =0 ;while (dio->bio_details.bi_iter.bi_size) {struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -const char *mem =bvec_kmap_local (&bv); +const char *mem =kmap_local_page (bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos);kunmap_local (mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT );
@@ -2626,9 +2626,8 @@ static int dm_integrity_end_io(struct dmwhile (dio->bio_details.bi_iter.bi_size) {char digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =bvec_kmap_local (&bv); -//memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, digest); +char *mem =kmap_local_page (bv.bv_page); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) {kunmap_local (mem);
Mikulas Patocka
maintainerPATCH 3/7 dm-integrity: introduce integrity_kmap and integrity_kunmap
In reply to Mikulas PatockaThis abstraction will be used later, for the asynchronous hash interface.
--- drivers/md/dm-integrity.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-)
drivers/md/dm-integrity.c +17 -20
@@ -1688,6 +1688,16 @@ failed:get_random_bytes (result, ic->tag_size); } +static void *integrity_kmap (struct dm_integrity_c *ic,struct page *p) +{ +return kmap_local_page (p); +} + +static void integrity_kunmap (struct dm_integrity_c *ic,const void *ptr) +{ +kunmap_local (ptr); +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) {struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));
@@ -1838,7 +1848,7 @@ static void integrity_metadata(struct wochar *mem, *checksums_ptr; again: - mem =kmap_local_page (bv_copy.bv_page); + mem =integrity_kmap (ic, bv_copy.bv_page); pos =0 ; checksums_ptr = checksums; do {
@@ -1848,7 +1858,7 @@ again: pos += ic->sectors_per_block <<SECTOR_SHIFT ; sector += ic->sectors_per_block; }while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack); -kunmap_local (mem); +integrity_kunmap (ic, mem); r =dm_integrity_rw_tag (ic, checksums, &dio->metadata_block, &dio->metadata_offset, checksums_ptr - checksums, dio->op ==REQ_OP_READ ?TAG_CMP :TAG_WRITE );
@@ -2072,19 +2082,6 @@ retry_kmap: js++; mem_ptr +=1 <<SECTOR_SHIFT ; }while (++s < ic->sectors_per_block); -#ifdef INTERNAL_VERIFY -if (ic->internal_hash) { -char checksums_onstack[MAX_T (size_t,HASH_MAX_DIGESTSIZE ,MAX_TAG_SIZE )]; - -integrity_sector_checksum (ic, logical_sector, mem + bv.bv_offset, checksums_onstack); -if (unlikely (crypto_memneq (checksums_onstack,journal_entry_tag (ic, je), ic->tag_size))) { -DMERR_LIMIT ("Checksum failed when reading from journal, at sector 0x%llx" , - logical_sector); -dm_audit_log_bio (DM_MSG_PREFIX ,"journal-checksum" , - bio, logical_sector,0 ); - } - } -#endif }if (!ic->internal_hash) {
@@ -2506,11 +2503,11 @@ skip_spinlock:unsigned pos =0 ;while (dio->bio_details.bi_iter.bi_size) {struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -const char *mem =kmap_local_page (bv.bv_page); +const char *mem =integrity_kmap (ic, bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos); -kunmap_local (mem); +integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); }
@@ -2626,17 +2623,17 @@ static int dm_integrity_end_io(struct dmwhile (dio->bio_details.bi_iter.bi_size) {char digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =kmap_local_page (bv.bv_page); +char *mem =integrity_kmap (ic, bv.bv_page);integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) { -kunmap_local (mem); +integrity_kunmap (ic, mem);dm_integrity_free_payload (dio);INIT_WORK (&dio->work, dm_integrity_inline_recheck);queue_work (ic->offload_wq, &dio->work);return DM_ENDIO_INCOMPLETE ; } -kunmap_local (mem); +integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); }
Mikulas Patocka
maintainerPATCH 4/7 dm-integrity: allocate the recalculate buffer with kmalloc
In reply to Mikulas PatockaAllocate the recalculate buffer with kmalloc rather than vmalloc. This will be needed later, for the simplification of the asynchronous hash interface.
--- drivers/md/dm-integrity.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/md/dm-integrity.c +4 -4
@@ -2998,7 +2998,7 @@ static void integrity_recalc(struct workunsigned recalc_sectors =RECALC_SECTORS ; retry: - recalc_buffer =__vmalloc (recalc_sectors <<SECTOR_SHIFT ,GFP_NOIO ); + recalc_buffer =kmalloc (recalc_sectors <<SECTOR_SHIFT ,GFP_NOIO | __GFP_NOWARN);if (!recalc_buffer) { oom: recalc_sectors >>=1 ;
@@ -3012,7 +3012,7 @@ oom: recalc_tags_size += ic->internal_hash_digestsize - ic->tag_size; recalc_tags =kvmalloc (recalc_tags_size,GFP_NOIO );if (!recalc_tags) { -vfree (recalc_buffer); +kfree (recalc_buffer); recalc_buffer =NULL ; goto oom; }
@@ -3078,7 +3078,7 @@ next_chunk: goto err; io_req.bi_opf =REQ_OP_READ ; - io_req.mem.type =DM_IO_VMA ; + io_req.mem.type =DM_IO_KMEM ; io_req.mem.ptr.addr = recalc_buffer; io_req.notify.fn =NULL ; io_req.client = ic->io;
@@ -3136,7 +3136,7 @@ unlock_ret:recalc_write_super (ic); free_ret: -vfree (recalc_buffer); +kfree (recalc_buffer);kvfree (recalc_tags); }
Mikulas Patocka
maintainerPATCH 5/7 dm-integrity: add the "offset" argument
In reply to Mikulas PatockaMake sure that the "data" argument passed to integrity_sector_checksum is always page-aligned and add an "offset" argument that specifies the offset from the start of the page. This will enable us to use the asynchronous hash interface later.
--- drivers/md/dm-integrity.c | 49 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-)
drivers/md/dm-integrity.c +35 -14
@@ -1636,7 +1636,7 @@ static void integrity_end_io(struct bio }static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, -const char *data,char *result) +const char *data,unsigned offset,char *result) {__le64 sector_le =cpu_to_le64 (sector);SHASH_DESC_ON_STACK (req, ic->internal_hash);
@@ -1665,7 +1665,7 @@ static void integrity_sector_checksum(st goto failed; } - r =crypto_shash_update (req, data, ic->sectors_per_block <<SECTOR_SHIFT ); + r =crypto_shash_update (req, data + offset, ic->sectors_per_block <<SECTOR_SHIFT );if (unlikely (r <0 )) {dm_integrity_io_error (ic,"crypto_shash_update" , r); goto failed;
@@ -1698,6 +1698,15 @@ static void integrity_kunmap(struct dm_ikunmap_local (ptr); } +static void *integrity_identity (struct dm_integrity_c *ic,void *data) +{ +#ifdef CONFIG_DEBUG_SG +BUG_ON (offset_in_page (data)); +BUG_ON (!virt_addr_valid (data)); +#endif +return data; +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) {struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io ));
@@ -1722,6 +1731,7 @@ static noinline void integrity_recheck(ssector_t alignment;char *mem;char *buffer =page_to_virt (page); +unsigned int buffer_offset;int r;struct dm_io_request io_req;struct dm_io_region io_loc;
@@ -1739,7 +1749,7 @@ static noinline void integrity_recheck(s alignment &= -alignment; io_loc.sector =round_down (io_loc.sector, alignment); io_loc.count += sector - io_loc.sector; - buffer += (sector - io_loc.sector) <<SECTOR_SHIFT ; + buffer_offset = (sector - io_loc.sector) <<SECTOR_SHIFT ; io_loc.count =round_up (io_loc.count, alignment); r =dm_io (&io_req,1 , &io_loc,NULL ,IOPRIO_DEFAULT );
@@ -1748,7 +1758,7 @@ static noinline void integrity_recheck(s goto free_ret; } -integrity_sector_checksum (ic, logical_sector, buffer, checksum); +integrity_sector_checksum (ic, logical_sector,integrity_identity (ic, buffer), buffer_offset, checksum); r =dm_integrity_rw_tag (ic, checksum, &dio->metadata_block, &dio->metadata_offset, ic->tag_size,TAG_CMP );if (r) {
@@ -1765,7 +1775,7 @@ static noinline void integrity_recheck(s } mem =bvec_kmap_local (&bv); -memcpy (mem + pos, buffer, ic->sectors_per_block <<SECTOR_SHIFT ); +memcpy (mem + pos, buffer + buffer_offset, ic->sectors_per_block <<SECTOR_SHIFT );kunmap_local (mem); pos += ic->sectors_per_block <<SECTOR_SHIFT ;
@@ -1852,7 +1862,7 @@ again: pos =0 ; checksums_ptr = checksums; do { - integrity_sector_checksum(ic, sector, mem + bv_copy.bv_offset + pos, checksums_ptr); + integrity_sector_checksum(ic, sector, mem, bv_copy.bv_offset + pos, checksums_ptr); checksums_ptr += ic->tag_size; sectors_to_process -= ic->sectors_per_block; pos += ic->sectors_per_block << SECTOR_SHIFT;
@@ -2123,14 +2133,16 @@ retry_kmap:if (ic->internal_hash) {unsigned int digest_size = ic->internal_hash_digestsize; +void *js_page =integrity_identity (ic, (char *)js -offset_in_page (js)); +unsigned js_offset =offset_in_page (js);if (unlikely (digest_size > ic->tag_size)) {char checksums_onstack[HASH_MAX_DIGESTSIZE ]; -integrity_sector_checksum (ic, logical_sector, (char *)js, checksums_onstack); +integrity_sector_checksum (ic, logical_sector, js_page, js_offset, checksums_onstack);memcpy (journal_entry_tag (ic, je), checksums_onstack, ic->tag_size); }else -integrity_sector_checksum (ic, logical_sector, (char *)js,journal_entry_tag (ic, je)); +integrity_sector_checksum (ic, logical_sector, js_page, js_offset,journal_entry_tag (ic, je)); }journal_entry_set_sector (je, logical_sector);
@@ -2506,7 +2518,7 @@ skip_spinlock:const char *mem =integrity_kmap (ic, bv.bv_page);if (ic->tag_size < ic->tuple_size)memset (dio->integrity_payload + pos + ic->tag_size,0 , ic->tuple_size - ic->tuple_size); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, dio->integrity_payload + pos); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos);integrity_kunmap (ic, mem); pos += ic->tuple_size;bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT );
@@ -2586,7 +2598,7 @@ static void dm_integrity_inline_recheck( }bio_put (outgoing_bio); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector,integrity_identity (ic, outgoing_data),0 , digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload,min (ic->internal_hash_digestsize, ic->tag_size)))) {DMERR_LIMIT ("%pg: Checksum failed at sector 0x%llx" , ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);
@@ -2624,7 +2636,7 @@ static int dm_integrity_end_io(struct dmchar digest[HASH_MAX_DIGESTSIZE ];struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter);char *mem =integrity_kmap (ic, bv.bv_page); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem + bv.bv_offset, digest); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest);if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos,min (ic->internal_hash_digestsize, ic->tag_size)))) {integrity_kunmap (ic, mem);
@@ -2899,9 +2911,12 @@ static void do_journal_write(struct dm_i#endif ic->internal_hash) {char test_tag[MAX_T (size_t,HASH_MAX_DIGESTSIZE ,MAX_TAG_SIZE )]; +struct journal_sector *js =access_journal_data (ic, i, l); +void *js_page =integrity_identity (ic, (char *)js -offset_in_page (js)); +unsigned js_offset =offset_in_page (js);integrity_sector_checksum (ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), - (char *)access_journal_data (ic, i, l), test_tag); + js_page, js_offset, test_tag);if (unlikely (crypto_memneq (test_tag,journal_entry_tag (ic, je2), ic->tag_size))) {dm_integrity_io_error (ic,"tag mismatch when replaying journal" , -EILSEQ );dm_audit_log_target (DM_MSG_PREFIX ,"integrity-replay-journal" , ic->ti,0 );
@@ -3094,7 +3109,10 @@ next_chunk: t = recalc_tags;for (i =0 ; i < n_sectors; i += ic->sectors_per_block) { -integrity_sector_checksum (ic, logical_sector + i, recalc_buffer + (i <<SECTOR_SHIFT ), t); +void *ptr = recalc_buffer + (i <<SECTOR_SHIFT ); +void *ptr_page =integrity_identity (ic, (char *)ptr -offset_in_page (ptr)); +unsigned ptr_offset =offset_in_page (ptr); +integrity_sector_checksum (ic, logical_sector + i, ptr_page, ptr_offset, t); t += ic->tag_size; }
@@ -3214,8 +3232,11 @@ next_chunk: t = recalc_tags;for (i =0 ; i < range.n_sectors; i += ic->sectors_per_block) { +void *ptr = recalc_buffer + (i <<SECTOR_SHIFT ); +void *ptr_page =integrity_identity (ic, (char *)ptr -offset_in_page (ptr)); +unsigned ptr_offset =offset_in_page (ptr);memset (t,0 , ic->tuple_size); -integrity_sector_checksum (ic, range.logical_sector + i, recalc_buffer + (i <<SECTOR_SHIFT ), t); +integrity_sector_checksum (ic, range.logical_sector + i, ptr_page, ptr_offset, t); t += ic->tuple_size; }
Mikulas Patocka
maintainerPATCH 6/7 dm-integrity: rename internal_hash
In reply to Mikulas PatockaRename "internal_hash" to "internal_shash" and introduce a boolean value "internal_hash".
--- drivers/md/dm-integrity.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-)
drivers/md/dm-integrity.c +11 -8
@@ -219,10 +219,11 @@ struct dm_integrity_c {__u8 log2_blocks_per_bitmap_bit;unsigned char mode; +bool internal_hash;int failed; -struct crypto_shash *internal_hash; +struct crypto_shash *internal_shash;unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1639,11 +1640,11 @@ static void integrity_sector_checksum(st const char *data, unsigned offset, char *result) { __le64 sector_le = cpu_to_le64(sector); -SHASH_DESC_ON_STACK (req, ic->internal_hash); +SHASH_DESC_ON_STACK (req, ic->internal_shash);int r;unsigned int digest_size; - req->tfm = ic->internal_hash; + req->tfm = ic->internal_shash; r =crypto_shash_init (req);if (unlikely (r <0 )) {
@@ -4708,12 +4709,14 @@ static int dm_integrity_ctr(struct dm_ta buffer_sectors =1 ; ic->log2_buffer_sectors =min ((int )__fls (buffer_sectors),31 -SECTOR_SHIFT ); - r =get_mac (&ic->internal_hash, &ic->internal_hash_alg, &ti->error, + r =get_mac (&ic->internal_shash, &ic->internal_hash_alg, &ti->error,"Invalid internal hash" ,"Error setting internal hash key" );if (r) goto bad; -if (ic->internal_hash) - ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_hash); +if (ic->internal_shash) { + ic->internal_hash = true; + ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_shash); + } r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );
@@ -5235,8 +5238,8 @@ static void dm_integrity_dtr(struct dm_tif (ic->sb)free_pages_exact (ic->sb,SB_SECTORS <<SECTOR_SHIFT ); -if (ic->internal_hash) -crypto_free_shash (ic->internal_hash); +if (ic->internal_shash) +crypto_free_shash (ic->internal_shash);free_alg (&ic->internal_hash_alg);if (ic->journal_crypt)
Mikulas Patocka
maintainerPATCH 7/7 dm-integrity: enable asynchronous hash interface
In reply to Mikulas PatockaThis commit enables the asynchronous hash interface in dm-integrity.
--- drivers/md/dm-integrity.c | 196 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 162 insertions(+), 34 deletions(-)
drivers/md/dm-integrity.c +162 -34
@@ -224,6 +224,7 @@ struct dm_integrity_c {int failed;struct crypto_shash *internal_shash; +struct crypto_ahash *internal_ahash;unsigned int internal_hash_digestsize;struct dm_target *ti;
@@ -1636,8 +1637,8 @@ static void integrity_end_io(struct biodec_in_flight (dio); } -static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, -const char *data,unsigned offset,char *result) +static void integrity_sector_checksum_shash (struct dm_integrity_c *ic,sector_t sector, +const char *data,unsigned offset,char *result) {__le64 sector_le =cpu_to_le64 (sector);SHASH_DESC_ON_STACK (req, ic->internal_shash);
@@ -1689,14 +1690,84 @@ failed:get_random_bytes (result, ic->tag_size); } +static void integrity_sector_checksum_ahash (struct dm_integrity_c *ic,sector_t sector, +struct page *page,unsigned offset,char *result) +{ +__le64 sector_le =cpu_to_le64 (sector); +HASH_REQUEST_ON_STACK (req, ic->internal_ahash); +DECLARE_CRYPTO_WAIT (wait); +struct scatterlist sg[3 ], *s = sg; +int r; +unsigned int digest_size; +unsigned int nbytes =0 ; + +might_sleep (); + + req->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK ; +ahash_request_set_callback (req,0 , crypto_req_done, &wait); + +if (ic->sb->flags &cpu_to_le32 (SB_FLAG_FIXED_HMAC )) { +sg_init_table (sg,3 ); +sg_set_buf (s, (const __u8 *)&ic->sb->salt,SALT_SIZE ); + nbytes +=SALT_SIZE ; + s++; + }else { +sg_init_table (sg,2 ); + } + +if (likely (!is_vmalloc_addr (§or_le))) { +sg_set_buf (s, §or_le,sizeof (sector_le)); + }else { +struct page *sec_page =vmalloc_to_page (§or_le); +unsigned int sec_off =offset_in_page (§or_le); +sg_set_page (s, sec_page,sizeof (sector_le), sec_off); + } + s++; + nbytes +=sizeof (sector_le); + +sg_set_page (s, page, ic->sectors_per_block <<SECTOR_SHIFT , offset); + nbytes += ic->sectors_per_block <<SECTOR_SHIFT ; + +ahash_request_set_crypt (req, sg, result, nbytes); + + r =crypto_wait_req (crypto_ahash_digest (req), &wait); +if (unlikely (r)) { +dm_integrity_io_error (ic,"crypto_ahash_digest" , r); + goto failed; + } + + digest_size = ic->internal_hash_digestsize; +if (unlikely (digest_size < ic->tag_size)) +memset (result + digest_size,0 , ic->tag_size - digest_size); + +return ; + +failed: +/* this shouldn't happen anyway, the hash functions have no reason to fail */ +get_random_bytes (result, ic->tag_size); +} + +static void integrity_sector_checksum (struct dm_integrity_c *ic,sector_t sector, +const char *data,unsigned offset,char *result) +{ +if (likely (ic->internal_shash !=NULL )) +integrity_sector_checksum_shash (ic, sector, data, offset, result); +else +integrity_sector_checksum_ahash (ic, sector, (struct page *)data, offset, result); +} +static void *integrity_kmap (struct dm_integrity_c *ic,struct page *p) { -return kmap_local_page (p); +if (likely (ic->internal_shash !=NULL )) +return kmap_local_page (p); +else +return p; }static void integrity_kunmap (struct dm_integrity_c *ic,const void *ptr) { -kunmap_local (ptr); +if (likely (ic->internal_shash !=NULL )) +kunmap_local (ptr); }static void *integrity_identity (struct dm_integrity_c *ic,void *data)
@@ -1705,7 +1776,10 @@ static void *integrity_identity(struct dBUG_ON (offset_in_page (data));BUG_ON (!virt_addr_valid (data));#endif -return data; +if (likely (ic->internal_shash !=NULL )) +return data; +else +return virt_to_page (data); } static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
@@ -2623,32 +2697,58 @@ static void dm_integrity_inline_recheck(bio_endio (bio); } +static inline bool dm_integrity_check (struct dm_integrity_c *ic,struct dm_integrity_io *dio) +{ +struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io )); +unsigned pos =0 ; + +while (dio->bio_details.bi_iter.bi_size) { +char digest[HASH_MAX_DIGESTSIZE ]; +struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); +char *mem =integrity_kmap (ic, bv.bv_page); +integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); +if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, +min (ic->internal_hash_digestsize, ic->tag_size)))) { +integrity_kunmap (ic, mem); +dm_integrity_free_payload (dio); +INIT_WORK (&dio->work, dm_integrity_inline_recheck); +queue_work (ic->offload_wq, &dio->work); +return false; + } +integrity_kunmap (ic, mem); + pos += ic->tuple_size; +bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); + } + +return true; +} + +static void dm_integrity_inline_async_check (struct work_struct *w) +{ + struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); +struct bio *bio =dm_bio_from_per_bio_data (dio,sizeof (struct dm_integrity_io )); +struct dm_integrity_c *ic = dio->ic; + +if (likely (dm_integrity_check (ic, dio))) +bio_endio (bio); +} +static int dm_integrity_end_io (struct dm_target *ti,struct bio *bio,blk_status_t *status) {struct dm_integrity_c *ic = ti->private;if (ic->mode =='I' ) {struct dm_integrity_io *dio =dm_per_bio_data (bio,sizeof (struct dm_integrity_io )); -if (dio->op ==REQ_OP_READ &&likely (*status ==BLK_STS_OK )) { -unsigned pos =0 ; +if (dio->op ==REQ_OP_READ &&likely (*status ==BLK_STS_OK ) &&likely (dio->bio_details.bi_iter.bi_size !=0 )) {if (ic->sb->flags &cpu_to_le32 (SB_FLAG_RECALCULATING ) &&unlikely (dio->integrity_range_locked)) - goto skip_check; -while (dio->bio_details.bi_iter.bi_size) { -char digest[HASH_MAX_DIGESTSIZE ]; -struct bio_vec bv =bio_iter_iovec (bio, dio->bio_details.bi_iter); -char *mem =integrity_kmap (ic, bv.bv_page); -integrity_sector_checksum (ic, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); -if (unlikely (crypto_memneq (digest, dio->integrity_payload + pos, -min (ic->internal_hash_digestsize, ic->tag_size)))) { -integrity_kunmap (ic, mem); -dm_integrity_free_payload (dio); -INIT_WORK (&dio->work, dm_integrity_inline_recheck); -queue_work (ic->offload_wq, &dio->work); + goto skip_check; +if (likely (ic->internal_shash !=NULL )) { +if (unlikely (!dm_integrity_check (ic, dio)))return DM_ENDIO_INCOMPLETE ; - } -integrity_kunmap (ic, mem); - pos += ic->tuple_size; -bio_advance_iter_single (bio, &dio->bio_details.bi_iter, ic->sectors_per_block <<SECTOR_SHIFT ); + }else { +INIT_WORK (&dio->work, dm_integrity_inline_async_check); +queue_work (ic->offload_wq, &dio->work); +return DM_ENDIO_INCOMPLETE ; } } skip_check:
@@ -4229,27 +4329,49 @@ nomem:return -ENOMEM ; } -static int get_mac (struct crypto_shash **hash,struct alg_spec *a,char **error, -char *error_alg,char *error_key) +static int get_mac (struct crypto_shash **shash,struct crypto_ahash **ahash, +struct alg_spec *a,char **error,char *error_alg,char *error_key) {int r;if (a->alg_string) { - *hash =crypto_alloc_shash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); -if (IS_ERR (*hash)) { +if (ahash) { + *ahash =crypto_alloc_ahash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); +if (IS_ERR (*ahash)) { + *ahash =NULL ; + goto try_shash; + } + +if (a->key) { + r =crypto_ahash_setkey (*ahash, a->key, a->key_size); +if (r) { + *error = error_key; +return r; + } + }else if (crypto_ahash_get_flags (*ahash) &CRYPTO_TFM_NEED_KEY ) { + *error = error_key; +return -ENOKEY ; + } + +return 0 ; + } + +try_shash: + *shash =crypto_alloc_shash (a->alg_string,0 ,CRYPTO_ALG_ALLOCATES_MEMORY ); +if (IS_ERR (*shash)) { *error = error_alg; - r =PTR_ERR (*hash); - *hash =NULL ; + r =PTR_ERR (*shash); + *shash =NULL ;return r; }if (a->key) { - r =crypto_shash_setkey (*hash, a->key, a->key_size); + r =crypto_shash_setkey (*shash, a->key, a->key_size);if (r) { *error = error_key;return r; } - }else if (crypto_shash_get_flags (*hash) &CRYPTO_TFM_NEED_KEY ) { + }else if (crypto_shash_get_flags (*shash) &CRYPTO_TFM_NEED_KEY ) { *error = error_key;return -ENOKEY ; }
@@ -4709,7 +4831,7 @@ static int dm_integrity_ctr(struct dm_ta buffer_sectors =1 ; ic->log2_buffer_sectors =min ((int )__fls (buffer_sectors),31 -SECTOR_SHIFT ); - r =get_mac (&ic->internal_shash, &ic->internal_hash_alg, &ti->error, + r =get_mac (&ic->internal_shash, &ic->internal_ahash, &ic->internal_hash_alg, &ti->error,"Invalid internal hash" ,"Error setting internal hash key" );if (r) goto bad;
@@ -4717,8 +4839,12 @@ static int dm_integrity_ctr(struct dm_ta ic->internal_hash = true; ic->internal_hash_digestsize =crypto_shash_digestsize (ic->internal_shash); } +if (ic->internal_ahash) { + ic->internal_hash = true; + ic->internal_hash_digestsize =crypto_ahash_digestsize (ic->internal_ahash); + } - r =get_mac (&ic->journal_mac, &ic->journal_mac_alg, &ti->error, + r =get_mac (&ic->journal_mac,NULL , &ic->journal_mac_alg, &ti->error,"Invalid journal mac" ,"Error setting journal mac key" );if (r) goto bad;
@@ -5240,6 +5366,8 @@ static void dm_integrity_dtr(struct dm_tif (ic->internal_shash)crypto_free_shash (ic->internal_shash); +if (ic->internal_ahash) +crypto_free_ahash (ic->internal_ahash);free_alg (&ic->internal_hash_alg);if (ic->journal_crypt)
@@ -5256,7 +5384,7 @@ static void dm_integrity_dtr(struct dm_tstatic struct target_type integrity_target = { .name ="integrity" , - .version = {1 ,13 ,0 }, + .version = {1 ,14 ,0 }, .module =THIS_MODULE , .features =DM_TARGET_SINGLETON |DM_TARGET_INTEGRITY , .ctr = dm_integrity_ctr,