libext2fs: Add merge/clone abstraction changes
cover libext2fs: Add merge/clone abstraction changes
Hello,
Though this series is not yet ready to be merged (due to few more todos), but I atleast wanted to get this base patch series out with some action items listed of what I am planning to do next and whether it is inline with the expectation as I last discussed here [1].
So looking at older review comments, it looks like we wanted a base patch series with libext2fs clone/merge apis so that anyone wanting to do parallel I/O could utilize libext2fs abstraction changes API for clone and merge of it's ext2_filsys structure.
Earlier the I/O manager abstraction changes for doing parallel I/O were already merged. This patch series mainly adds the libext2fs abstraction changes for cloning and merging libext2fs data structures.
So next few todos that I am planning are (some of which are sitting in my tree but still needs more work)
1. Add dblist unit test - there is nothing that exist today for dblist unit
tests.
2. Fix the todos listed in libext2fs merge/clone patch since some of those are not required while cloning ext2_filsys structure and it's members.
3. Start bringing in the pfsck base changes (work done by others) to this series along with writing unit tests for those.
4. Work on making older pfsck patches, work with latest features that have gone in like fastcommit, casefolding etc.
5. Make sure that the later bug fixes (and some which I have identified), should be included in the original feature patches (rather than as a separate bug fix patches).
6. Make sure to add more tests for pfsck with fuzzed images with e2fuzz.
7. Later we could work on other passes and/or other tools like debugfs etc, to make it parallel.
Please let me know if this is going in the right direction. Also if you could take a look at this series and kindly let me know whether patches looks ok to you.
I guess this might have taken longer than I earlier estimated. But I will definitely try to correct that with future revisions.
[1]: https://lore.kernel.org/linux-ext4/20220321164009.dwqmdo7axyyixn2t@riteshh-domain/
-ritesh
NOTE (Background)
===================
As I understand, earlier to make fsck parallel, the patches added io_manager relationship directly within e2fsck_t struct. Which is a layering violation which will expose internal library functions and structures to it's clients. Also if any other client (e.g. debugfs etc.) who would like to add threading support will also have to do the same work in their structures.
So instead we now have added a parent structure of same type within ext2_filsys struct for multi-threading support. And libext2fs is responsible for providing the meaningful apis to it's client for clone/merge of it's data structures/bitmaps for parallel threading support based on the flags passed by it's clients.
+------------------+
| DS relationships | %===============================================% % typedef struct struct_io_manager *io_manager; % % typedef struct struct_io_channel *io_channel; %
% typedef struct struct_io_stats *io_stats; %
%===============================================%
+-----------------+
| e2fsck/e2fsck.h |
+-------------------------------+
| (ctx) struct e2fsck_t { |
+------|------ ext2_filsys fs |
| | io_manager io_manager --|-------------------+ --> layering violation
| | io_channel journal_io | |
| | } | |
| +-------------------------------+ |
+---+ |
<<<< Libext2fs >>>> | |
+----------------------+ |
| lib/ext2fs/ext2fs.h | |
+-----------------------------+ |
| (fs) struct ext2_filsys { | |
| (struct struct_ext2_filsys) | |
| io_channel io | |
| | | |
+--------------v--------------+ |
| |
<<<< IO LAYER >>>>>>> | |
+----------------------+ |
| lib/ext2fs/ext2_io.h | |
+-----------------------------------------------+----+ |
| (io) typedef struct struct_io_channel *io_channel{ | |
+---- | io_manager manager | |
| | void *private_data | |
| +----------------------------------------------------+ |
| | |
| | (io_channel-> io_manager) v
| +--------------------------------------------------------------------------------------------------+ | | lib/ext2fs/windows_io.c <global> 1041 io_manager windows_io_manager = &struct_windows_manager; |
| | lib/ext2fs/sparse_io.c <global> 554 io_manager sparsefd_io_manager = &struct_sparsefd_manager; |
| | lib/ext2fs/sparse_io.c <global> 553 io_manager sparse_io_manager = &struct_sparse_manager; |
| | lib/ext2fs/unix_io.c <global> 1437 io_manager unixfd_io_manager = &struct_unixfd_manager; |
| | lib/ext2fs/inode_io.c <global> 79 io_manager inode_io_manager = &struct_inode_manager; |
| | lib/ext2fs/unix_io.c <global> 1415 io_manager unix_io_manager = &struct_unix_manager; |
| | lib/ext2fs/undo_io.c <global> 1125 io_manager undo_io_manager = &struct_undo_manager; |
| | lib/ext2fs/test_io.c <global> 555 io_manager test_io_manager = &struct_test_manager; |
| | lib/ext2fs/dosio.c <global> 75 io_manager dos_io_manager = &struct_dos_manager; |
| +--------------------------------------------------------------------------------------------------+ |
| (io_channel->private_data)
| +-------------------------------------------------------------+
| | lib/ext2fs/nt_io.c:206:20:typedef struct _NT_PRIVATE_DATA { |
| | lib/ext2fs/windows_io.c:95:16:struct windows_private_data { |
| | lib/ext2fs/inode_io.c:34:14:struct inode_private_data { | +--------------------------------------+
--->| lib/ext2fs/unix_io.c:103:13:struct unix_private_data { |<-------| struct unix_private_data { |
| lib/ext2fs/undo_io.c:126:13:struct undo_private_data { | | struct struct_io_stats io_stats; |
| lib/ext2fs/test_io.c:45:13:struct test_private_data { | +--------------------------------------+
+------------------------^------------------------------------+
|
+-------------------|--------------------+
| struct undo_private_data { |
| /* the undo file io channel */ |
| io_channel undo_file; |
| /* The backing io channel */ |
| io_channel real; |
| char *tdb_file; |
| } |
+----------------------------------------+
Fig: Depicting that with this patch series, there is no need to add "io_manager" into ext2_filsys which was done by original pfsck implementation.
Li Xi (1):
dblist: add dblist merge logic
Ritesh Harjani (8):
gen_bitmaps: Fix ext2fs_compare_generic_bmap/bitmap logic
badblocks: Remove unused badblocks_flags
blkmap64_ba: Add common helper for bits size calculation
blkmap64_ba: Implement initial implementation of merge bitmaps
tst_bitmaps_standalone: Add copy and merge bitmaps test
tst_bitmaps_pthread: Add merge bitmaps test using pthreads
tst_badblocks: Add unit test to verify badblocks list merge api
tst_libext2fs_pthread: Add libext2fs merge/clone unit tests
Saranya Muruganandam (1):
libext2fs: dupfs: Add fs clone & merge api
Wang Shilong (3):
ext2fs/bitmaps: Add merge bitmaps library abstraction changes
libext2fs: blkmap64_rb: Add rbtree bmap merge logic changes
badblocks: Add badblocks merge logic
lib/ext2fs/Makefile.in | 53 ++++- lib/ext2fs/badblocks.c | 81 ++++++- lib/ext2fs/bitmaps.c | 9 + lib/ext2fs/blkmap64_ba.c | 73 ++++++- lib/ext2fs/blkmap64_rb.c | 65 ++++++ lib/ext2fs/bmap64.h | 5 + lib/ext2fs/dblist.c | 36 ++++ lib/ext2fs/dupfs.c | 149 +++++++++++++ lib/ext2fs/ext2fs.h | 35 +++ lib/ext2fs/ext2fsP.h | 1 - lib/ext2fs/gen_bitmap.c | 9 +- lib/ext2fs/gen_bitmap64.c | 39 +++- lib/ext2fs/tst_badblocks.c | 61 +++++- lib/ext2fs/tst_bitmaps_pthread.c | 247 +++++++++++++++++++++ lib/ext2fs/tst_bitmaps_standalone.c | 173 +++++++++++++++ lib/ext2fs/tst_libext2fs_pthread.c | 322 ++++++++++++++++++++++++++++ 16 files changed, 1330 insertions(+), 28 deletions(-) create mode 100644 lib/ext2fs/tst_bitmaps_pthread.c create mode 100644 lib/ext2fs/tst_bitmaps_standalone.c create mode 100644 lib/ext2fs/tst_libext2fs_pthread.c
--
2.35.3
RFC 1/13 gen_bitmaps: Fix ext2fs_compare_generic_bmap/bitmap logic
Currently this function was not correctly comparing against the right length of the bitmap. Also when we compare bitarray v/s rbtree bitmap the value returned by ext2fs_test_generic_bmap() could be different in these two implementations. Hence only check against boolean value.
--- lib/ext2fs/gen_bitmap.c | 9 ++++++--- lib/ext2fs/gen_bitmap64.c | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-)
lib/ext2fs/gen_bitmap.c +6 -3
@@ -385,10 +385,13 @@ errcode_t ext2fs_compare_generic_bitmap(errcode_t magic, errcode_t neq, (size_t) (bm1->end - bm1->start)/8)))return neq; -for (i = bm1->end - ((bm1->end - bm1->start) %8 ); i <= bm1->end; i++) -if (ext2fs_fast_test_block_bitmap (gen_bm1, i) != -ext2fs_fast_test_block_bitmap (gen_bm2, i)) +for (i = bm1->start; i <= bm1->end; i++) { +int ret1, ret2; + ret1 = !!ext2fs_fast_test_block_bitmap (gen_bm1, i); + ret2 = !!ext2fs_fast_test_block_bitmap (gen_bm2, i); +if (ret1 != ret2)return neq; + }return 0 ; }
lib/ext2fs/gen_bitmap64.c +7 -3
@@ -629,10 +629,14 @@ errcode_t ext2fs_compare_generic_bmap(errcode_t neq, (bm1->end != bm2->end))return neq; -for (i = bm1->end - ((bm1->end - bm1->start) %8 ); i <= bm1->end; i++) -if (ext2fs_test_generic_bmap (gen_bm1, i) != -ext2fs_test_generic_bmap (gen_bm2, i)) +for (i = bm1->start; i < bm1->end; i++) { +int ret1, ret2; + ret1 = !!ext2fs_test_generic_bmap (gen_bm1, i); + ret2 = !!ext2fs_test_generic_bmap (gen_bm2, i); +if (ret1 != ret2) {return neq; + } + }return 0 ; }
Signature · 1 line
2.35.3
RFC 2/13 badblocks: Remove unused badblocks_flags
In reply to Ritesh Harjanibadblocks_flags is not used anywhere. So just remove it.
--- lib/ext2fs/badblocks.c | 6 +----- lib/ext2fs/ext2fsP.h | 1 - 2 files changed, 1 insertion(+), 6 deletions(-)
lib/ext2fs/badblocks.c +1 -5
@@ -81,11 +81,7 @@ errcode_t ext2fs_u32_copy(ext2_u32_list src, ext2_u32_list *dest) {errcode_t retval; - retval =make_u32_list (src->size, src->num, src->list, dest); -if (retval) -return retval; - (*dest)->badblocks_flags = src->badblocks_flags; -return 0 ; +return make_u32_list (src->size, src->num, src->list, dest); } errcode_t ext2fs_badblocks_copy(ext2_badblocks_list src,
lib/ext2fs/ext2fsP.h +0 -1
@@ -34,7 +34,6 @@ struct ext2_struct_u32_list {int num;int size;__u32 *list; -int badblocks_flags; };struct ext2_struct_u32_iterate {
Signature · 1 line
2.35.3
RFC 3/13 blkmap64_ba: Add common helper for bits size calculation
In reply to Ritesh HarjaniJust a quick common helper for bits size calculation.
--- lib/ext2fs/blkmap64_ba.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-)
lib/ext2fs/blkmap64_ba.c +13 -7
@@ -40,6 +40,13 @@ struct ext2fs_ba_private_struct {typedef struct ext2fs_ba_private_struct *ext2fs_ba_private ; +#define ba_bits_size (start, end) ((((end) - (start)) / 8 + 1)) + +static size_t ba_bitmap_size (ext2fs_generic_bitmap_64 bitmap) +{ +return (size_t )ba_bits_size (bitmap->start, bitmap->real_end); +} +static errcode_t ba_alloc_private_data (ext2fs_generic_bitmap_64 bitmap) {ext2fs_ba_private bp;
@@ -56,7 +63,7 @@ static errcode_t ba_alloc_private_data (ext2fs_generic_bitmap_64 bitmap)if (retval)return retval; - size = (size_t ) (((bitmap->real_end - bitmap->start) /8 ) +1 ); + size =ba_bitmap_size (bitmap); retval =ext2fs_get_mem (size, &bp->bitarray);if (retval) {
@@ -80,7 +87,7 @@ static errcode_t ba_new_bmap(ext2_filsys fs EXT2FS_ATTR((unused)),return retval; bp = (ext2fs_ba_private ) bitmap->private; - size = (size_t ) (((bitmap->real_end - bitmap->start) /8 ) +1 ); + size =ba_bitmap_size (bitmap);memset (bp->bitarray,0 , size);return 0 ;
@@ -115,7 +122,7 @@ static errcode_t ba_copy_bmap(ext2fs_generic_bitmap_64 src, dest_bp = (ext2fs_ba_private ) dest->private; - size = (size_t ) (((src->real_end - src->start) /8 ) +1 ); + size =ba_bitmap_size (src);memcpy (dest_bp->bitarray, src_bp->bitarray, size);return 0 ;
@@ -145,8 +152,8 @@ static errcode_t ba_resize_bmap(ext2fs_generic_bitmap_64 bmap,return 0 ; } - size = ((bmap->real_end - bmap->start) /8 ) +1 ; - new_size = ((new_real_end - bmap->start) /8 ) +1 ; + size =ba_bitmap_size (bmap); + new_size =ba_bits_size (new_real_end, bmap->start);if (size != new_size) { retval =ext2fs_resize_mem (size, new_size, &bp->bitarray);
@@ -306,8 +313,7 @@ static void ba_clear_bmap(ext2fs_generic_bitmap_64 bitmap) {ext2fs_ba_private bp = (ext2fs_ba_private ) bitmap->private; -memset (bp->bitarray,0 , - (size_t ) (((bitmap->real_end - bitmap->start) /8 ) +1 )); +memset (bp->bitarray,0 ,ba_bitmap_size (bitmap)); }#ifdef ENABLE_BMAP_STATS
Signature · 1 line
2.35.3
RFC 4/13 ext2fs/bitmaps: Add merge bitmaps library abstraction changes
In reply to Ritesh HarjaniFrom: Wang Shilong <[email protected]>
Add merge bitmaps library abstraction changes.
--- lib/ext2fs/bitmaps.c | 9 +++++++++ lib/ext2fs/bmap64.h | 5 +++++ lib/ext2fs/ext2fs.h | 8 ++++++++ lib/ext2fs/gen_bitmap64.c | 29 +++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+)
lib/ext2fs/bitmaps.c +9 -0
@@ -45,6 +45,15 @@ errcode_t ext2fs_copy_bitmap(ext2fs_generic_bitmap src, {return (ext2fs_copy_generic_bmap (src, dest)); } + +errcode_t ext2fs_merge_bitmap (ext2fs_generic_bitmap src, +ext2fs_generic_bitmap dst, +ext2fs_generic_bitmap dup, +ext2fs_generic_bitmap dup_allowed) +{ +return ext2fs_merge_generic_bmap (src, dst, dup, dup_allowed); +} +void ext2fs_set_bitmap_padding (ext2fs_generic_bitmap map) {ext2fs_set_generic_bmap_padding (map);
lib/ext2fs/bmap64.h +5 -0
@@ -100,6 +100,11 @@ struct ext2_bitmap_ops {* May be NULL, in which case a generic function is used. */ errcode_t (*find_first_set)(ext2fs_generic_bitmap_64 bitmap,__u64 start,__u64 end,__u64 *out); + +errcode_t (*merge_bmap)(ext2fs_generic_bitmap_64 src, +ext2fs_generic_bitmap_64 dest, +ext2fs_generic_bitmap_64 dup, +ext2fs_generic_bitmap_64 dup_allowed); };extern struct ext2_bitmap_ops ext2fs_blkmap64_bitarray;
lib/ext2fs/ext2fs.h +8 -0
@@ -867,6 +867,10 @@ extern void ext2fs_free_block_bitmap(ext2fs_block_bitmap bitmap);extern void ext2fs_free_inode_bitmap (ext2fs_inode_bitmap bitmap);extern errcode_t ext2fs_copy_bitmap (ext2fs_generic_bitmap src,ext2fs_generic_bitmap *dest); +extern errcode_t ext2fs_merge_bitmap (ext2fs_generic_bitmap src, +ext2fs_generic_bitmap dst, +ext2fs_generic_bitmap dup, +ext2fs_generic_bitmap dup_allowed);extern errcode_t ext2fs_allocate_block_bitmap (ext2_filsys fs,const char *descr,ext2fs_block_bitmap *ret);
@@ -1455,6 +1459,10 @@ errcode_t ext2fs_alloc_generic_bmap(ext2_filsys fs, errcode_t magic, ext2fs_generic_bitmap *ret);errcode_t ext2fs_copy_generic_bmap (ext2fs_generic_bitmap src,ext2fs_generic_bitmap *dest); +extern errcode_t ext2fs_merge_generic_bmap (ext2fs_generic_bitmap gen_src, +ext2fs_generic_bitmap gen_dst, +ext2fs_generic_bitmap gen_dup, +ext2fs_generic_bitmap gen_dup_allowed);void ext2fs_clear_generic_bmap (ext2fs_generic_bitmap bitmap); errcode_t ext2fs_fudge_generic_bmap_end(ext2fs_generic_bitmap bitmap, errcode_t neq,
lib/ext2fs/gen_bitmap64.c +29 -0
@@ -346,6 +346,35 @@ errcode_t ext2fs_copy_generic_bmap(ext2fs_generic_bitmap gen_src,return 0 ; } +errcode_t ext2fs_merge_generic_bmap (ext2fs_generic_bitmap gen_src, +ext2fs_generic_bitmap gen_dst, +ext2fs_generic_bitmap gen_dup, +ext2fs_generic_bitmap gen_dup_allowed) +{ +ext2fs_generic_bitmap_64 src = (ext2fs_generic_bitmap_64 ) gen_src; +ext2fs_generic_bitmap_64 dst = (ext2fs_generic_bitmap_64 ) gen_dst; +ext2fs_generic_bitmap_64 dup = (ext2fs_generic_bitmap_64 ) gen_dup; +ext2fs_generic_bitmap_64 dup_allowed = (ext2fs_generic_bitmap_64 ) gen_dup_allowed; + +if (!src || !dst) +return EINVAL ; + +if (!EXT2FS_IS_64_BITMAP (src) || !EXT2FS_IS_64_BITMAP (dst) || + (dup && !EXT2FS_IS_64_BITMAP (dup)) || + (dup_allowed && !EXT2FS_IS_64_BITMAP (dup_allowed))) +return EINVAL ; + +if (src->bitmap_ops != dst->bitmap_ops || + (dup && dup->bitmap_ops != src->bitmap_ops) || + (dup_allowed && dup_allowed->bitmap_ops != src->bitmap_ops)) +return EINVAL ; + +if (!src->bitmap_ops->merge_bmap) +return EOPNOTSUPP ; + +return src->bitmap_ops->merge_bmap (src, dst, dup, dup_allowed); +} +errcode_t ext2fs_resize_generic_bmap (ext2fs_generic_bitmap gen_bmap,__u64 new_end,__u64 new_real_end)
Signature · 1 line
2.35.3
RFC 5/13 libext2fs: blkmap64_rb: Add rbtree bmap merge logic changes
In reply to Ritesh HarjaniFrom: Wang Shilong <[email protected]>
Add rbtree bmap merge logic changes.
--- lib/ext2fs/blkmap64_rb.c | 65 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+)
lib/ext2fs/blkmap64_rb.c +65 -0
@@ -977,11 +977,76 @@ static void rb_print_stats(ext2fs_generic_bitmap_64 bitmap EXT2FS_ATTR((unused)) }#endif +static errcode_t rb_merge_bmap (ext2fs_generic_bitmap_64 src, +ext2fs_generic_bitmap_64 dest, +ext2fs_generic_bitmap_64 dup, +ext2fs_generic_bitmap_64 dup_allowed) +{ +struct ext2fs_rb_private *src_bp, *dest_bp, *dup_bp =NULL ; +struct bmap_rb_extent *src_ext; +struct rb_node *src_node; +errcode_t retval =0 ; +int dup_found =0 ; +__u64 i; + + src_bp = (struct ext2fs_rb_private *) src->private; + dest_bp = (struct ext2fs_rb_private *) dest->private; +if (dup) + dup_bp = (struct ext2fs_rb_private *)dup->private; + src_bp->rcursor =NULL ; + dest_bp->rcursor =NULL ; + + src_node =ext2fs_rb_first (&src_bp->root); +while (src_node) { + src_ext =node_to_extent (src_node); + retval =rb_test_clear_bmap_extent (dest, + src_ext->start + src->start, + src_ext->count); +if (retval) { +rb_insert_extent (src_ext->start, src_ext->count, + dest_bp); + goto next; + } + +/* unlikely case, do it one by one block */ +for (i = src_ext->start; + i < src_ext->start + src_ext->count; i++) { + retval =rb_test_clear_bmap_extent (dest, i + src->start,1 ); +if (retval) { +rb_insert_extent (i,1 , dest_bp); +continue ; + } +if (dup_allowed) { + retval =rb_test_clear_bmap_extent (dup_allowed, + i + src->start,1 ); +/* not existed in dup_allowed */ +if (retval) { + dup_found =1 ; +if (dup_bp) +rb_insert_extent (i,1 , dup_bp); + }/* else we conside it not duplicated */ + }else { +if (dup_bp) +rb_insert_extent (i,1 , dup_bp); + dup_found =1 ; + } + } +next: + src_node =ext2fs_rb_next (src_node); + } + +if (dup_found && dup) +return EEXIST ; + +return 0 ; +} +struct ext2_bitmap_ops ext2fs_blkmap64_rbtree = { .type =EXT2FS_BMAP64_RBTREE , .new_bmap = rb_new_bmap, .free_bmap = rb_free_bmap, .copy_bmap = rb_copy_bmap, + .merge_bmap = rb_merge_bmap, .resize_bmap = rb_resize_bmap, .mark_bmap = rb_mark_bmap, .unmark_bmap = rb_unmark_bmap,
Signature · 1 line
2.35.3
RFC 6/13 blkmap64_ba: Implement initial implementation of merge bitmaps
In reply to Ritesh HarjaniDO NOT MERGE
Adding a basic merge implementation of bitarray for later adding/supporting test cases w.r.t. libext2fs merge/clone API.
--- lib/ext2fs/blkmap64_ba.c | 53 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-)
lib/ext2fs/blkmap64_ba.c +52 -1
@@ -12,6 +12,7 @@#include "config.h" #include <stdio.h> #include <string.h> +#include <assert.h> #if HAVE_UNISTD_H #include <unistd.h> #endif
@@ -476,6 +477,55 @@ static errcode_t ba_find_first_set(ext2fs_generic_bitmap_64 bitmap,return ENOENT ; } +errcode_t ba_merge_bmap (ext2fs_generic_bitmap_64 src, +ext2fs_generic_bitmap_64 dst, +ext2fs_generic_bitmap_64 dup, +ext2fs_generic_bitmap_64 dup_allowed) +{ +ext2fs_ba_private src_bp = (ext2fs_ba_private ) src->private; +ext2fs_ba_private dst_bp = (ext2fs_ba_private ) dst->private; + +const unsigned char *src_pos = src_bp->bitarray; +const unsigned char *dst_pos = dst_bp->bitarray; +unsigned long count = src->real_end - src->start +1 ; +unsigned long bitpos = src->start; + +assert (src->start == dst->start); +assert (src->end == dst->end); +assert (src->real_end == dst->real_end); + +// TODO add full support +// For now assuming the pos is aligned addr +assert (!(((uintptr_t )src_pos) &0x07 )); + +// 8-byte blocks compare +while (count >=64 ) { +const __u64 src_val = *(const __u64 *)src_pos; +const __u64 dst_val = *(const __u64 *)dst_pos; +const __u64 sd_val = src_val & dst_val; + +// TODO: Not implemented case to handle duplicates/dup_allowed case of EA +if (dup || dup_allowed) +assert (sd_val ==0 ); + + *(__u64 *)dst_pos |= src_val; + + src_pos +=8 ; + dst_pos +=8 ; + count -=64 ; + bitpos +=64 ; + } + +while (count-- >0 ) { +// TODO: dup case not implemented yet. +if (ext2fs_test_bit64 (bitpos, src_bp->bitarray)) +assert (ext2fs_set_bit64 (bitpos, dst_bp->bitarray) ==0 ); + bitpos++; + } + +return 0 ; +} +struct ext2_bitmap_ops ext2fs_blkmap64_bitarray = { .type =EXT2FS_BMAP64_BITARRAY ,
@@ -494,5 +544,6 @@ struct ext2_bitmap_ops ext2fs_blkmap64_bitarray = { .clear_bmap = ba_clear_bmap, .print_stats = ba_print_stats, .find_first_zero = ba_find_first_zero, - .find_first_set = ba_find_first_set + .find_first_set = ba_find_first_set, + .merge_bmap = ba_merge_bmap, };
Signature · 1 line
2.35.3
RFC 7/13 tst_bitmaps_standalone: Add copy and merge bitmaps test
In reply to Ritesh HarjaniThis adds a basic copy and merge api test for both bitmap types (i.e. rbtree and bitarray)
--- lib/ext2fs/Makefile.in | 25 +++- lib/ext2fs/tst_bitmaps_standalone.c | 173 ++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 lib/ext2fs/tst_bitmaps_standalone.c
lib/ext2fs/Makefile.in +19 -6
@@ -227,6 +227,7 @@ SRCS= ext2_err.c \ $(srcdir)/write_bb_file.c \ $(srcdir)/rbtree.c \ $(srcdir)/tst_libext2fs.c \ + $(srcdir)/tst_bitmaps_standalone.c \ $(DEBUG_SRCS) HFILES= bitops.h ext2fs.h ext2_io.h ext2_fs.h ext2_ext_attr.h ext3_extents.h \
@@ -328,9 +329,9 @@ tst_getsectsize: tst_getsectsize.o getsectsize.o $(STATIC_LIBEXT2FS) \ $(ALL_LDFLAGS) $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) \ $(SYSLIBS) -tst_types.o: $(srcdir)/tst_types.c ext2_types.h +tst_types.o: $(srcdir)/tst_types.c ext2_types.h -tst_types: tst_types.o ext2_types.h +tst_types: tst_types.o ext2_types.h $(E) " LD $@" $(Q) $(CC) -o tst_types tst_types.o $(ALL_LDFLAGS) $(SYSLIBS)
@@ -362,6 +363,11 @@ tst_sha512: $(srcdir)/sha512.c $(srcdir)/ext2_fs.h $(Q) $(CC) $(ALL_LDFLAGS) $(ALL_CFLAGS) -o tst_sha512 \ $(srcdir)/sha512.c -DUNITTEST $(SYSLIBS) +tst_bitmaps_standalone: tst_bitmaps_standalone.o $(STATIC_LIBEXT2FS) $(DEPSTATIC_LIBCOM_ERR) + $(E) " LD $@" + $(Q) $(CC) -o tst_bitmaps_standalone tst_bitmaps_standalone.o $(ALL_LDFLAGS) \ + $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) $(SYSLIBS) + ext2_tdbtool: tdbtool.o $(E) " LD $@" $(Q) $(CC) -o ext2_tdbtool tdbtool.o tdb.o $(ALL_LDFLAGS) $(SYSLIBS)
@@ -533,7 +539,7 @@ mkjournal: mkjournal.c $(STATIC_LIBEXT2FS) $(DEPLIBCOM_ERR) fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ tst_super_size tst_types tst_inode_size tst_csum tst_crc32c tst_bitmaps \ tst_inline tst_inline_data tst_libext2fs tst_sha256 tst_sha512 \ - tst_digest_encode tst_getsize tst_getsectsize + tst_digest_encode tst_getsize tst_getsectsize tst_bitmaps_standalone $(TESTENV) ./tst_bitops $(TESTENV) ./tst_badblocks $(TESTENV) ./tst_iscan
@@ -556,6 +562,7 @@ fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ $(TESTENV) ./tst_bitmaps -l -f $(srcdir)/tst_bitmaps_cmds > tst_bitmaps_out diff $(srcdir)/tst_bitmaps_exp tst_bitmaps_out $(TESTENV) ./tst_digest_encode + $(TESTENV) ./tst_bitmaps_standalone installdirs:: $(E) " MKDIR_P $(libdir) $(includedir)/ext2fs"
@@ -581,7 +588,7 @@ install:: all $(HFILES) $(HFILES_IN) installdirs ext2fs.pc uninstall:: $(RM) -f $(DESTDIR)$(libdir)/libext2fs.a \ $(DESTDIR)$(pkgconfigdir)/ext2fs.pc - $(RM) -rf $(DESTDIR)$(includedir)/ext2fs + $(RM) -rf $(DESTDIR)$(includedir)/ext2fs clean:: $(RM) -f \#* *.s *.o *.a *~ *.bak core profiled/* \
@@ -590,7 +597,7 @@ clean:: tst_bitops tst_types tst_icount tst_super_size tst_csum \ tst_bitmaps tst_bitmaps_out tst_extents tst_inline \ tst_inline_data tst_inode_size tst_bitmaps_cmd.c \ - tst_digest_encode tst_sha256 tst_sha512 \ + tst_digest_encode tst_sha256 tst_sha512 tst_bitmaps_standalone \ ext2_tdbtool mkjournal debug_cmds.c tst_cmds.c extent_cmds.c \ ../libext2fs.a ../libext2fs_p.a ../libext2fs_chk.a \ crc32c_table.h gen_crc32ctable tst_crc32c tst_libext2fs \
@@ -646,7 +653,7 @@ windows_io.o: $(srcdir)/windows_io.c $(top_builddir)/lib/config.h \ $(srcdir)/ext2_ext_attr.h $(srcdir)/bitops.h $(srcdir)/ext2fsP.h # +++ Dependency line eater +++ -# +# # Makefile dependencies follow. This must be the last section in # the Makefile.in file #
@@ -1156,6 +1163,12 @@ tst_iscan.o: $(srcdir)/tst_iscan.c $(top_builddir)/lib/config.h \ $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h +tst_bitmaps_standalone.o: $(srcdir)/tst_bitmaps_standalone.c $(top_builddir)/lib/config.h \ + $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ + $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \ + $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ + $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ + $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h undo_io.o: $(srcdir)/undo_io.c $(top_builddir)/lib/config.h \ $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \
lib/ext2fs/tst_bitmaps_standalone.c new file +173 -0
173 lines not on this page · raw message
Signature · 1 line
2.35.3
RFC 8/13 tst_bitmaps_pthread: Add merge bitmaps test using pthreads
In reply to Ritesh HarjaniThis patch adds a test to verify the core bitmaps merge APIs for both bitarray and rbtree type.
--- lib/ext2fs/Makefile.in | 17 ++- lib/ext2fs/tst_bitmaps_pthread.c | 247 +++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 lib/ext2fs/tst_bitmaps_pthread.c
lib/ext2fs/Makefile.in +16 -1
@@ -228,6 +228,7 @@ SRCS= ext2_err.c \ $(srcdir)/rbtree.c \ $(srcdir)/tst_libext2fs.c \ $(srcdir)/tst_bitmaps_standalone.c \ + $(srcdir)/tst_bitmaps_pthread.c \ $(DEBUG_SRCS) HFILES= bitops.h ext2fs.h ext2_io.h ext2_fs.h ext2_ext_attr.h ext3_extents.h \
@@ -368,6 +369,11 @@ tst_bitmaps_standalone: tst_bitmaps_standalone.o $(STATIC_LIBEXT2FS) $(DEPSTATIC $(Q) $(CC) -o tst_bitmaps_standalone tst_bitmaps_standalone.o $(ALL_LDFLAGS) \ $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) $(SYSLIBS) +tst_bitmaps_pthread: tst_bitmaps_pthread.o $(STATIC_LIBEXT2FS) $(DEPSTATIC_LIBCOM_ERR) + $(E) " LD $@" + $(Q) $(CC) -o tst_bitmaps_pthread tst_bitmaps_pthread.o $(ALL_LDFLAGS) \ + $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) $(SYSLIBS) + ext2_tdbtool: tdbtool.o $(E) " LD $@" $(Q) $(CC) -o ext2_tdbtool tdbtool.o tdb.o $(ALL_LDFLAGS) $(SYSLIBS)
@@ -539,7 +545,8 @@ mkjournal: mkjournal.c $(STATIC_LIBEXT2FS) $(DEPLIBCOM_ERR) fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ tst_super_size tst_types tst_inode_size tst_csum tst_crc32c tst_bitmaps \ tst_inline tst_inline_data tst_libext2fs tst_sha256 tst_sha512 \ - tst_digest_encode tst_getsize tst_getsectsize tst_bitmaps_standalone + tst_digest_encode tst_getsize tst_getsectsize tst_bitmaps_standalone \ + tst_bitmaps_pthread $(TESTENV) ./tst_bitops $(TESTENV) ./tst_badblocks $(TESTENV) ./tst_iscan
@@ -563,6 +570,7 @@ fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ diff $(srcdir)/tst_bitmaps_exp tst_bitmaps_out $(TESTENV) ./tst_digest_encode $(TESTENV) ./tst_bitmaps_standalone + $(TESTENV) ./tst_bitmaps_pthread installdirs:: $(E) " MKDIR_P $(libdir) $(includedir)/ext2fs"
@@ -598,6 +606,7 @@ clean:: tst_bitmaps tst_bitmaps_out tst_extents tst_inline \ tst_inline_data tst_inode_size tst_bitmaps_cmd.c \ tst_digest_encode tst_sha256 tst_sha512 tst_bitmaps_standalone \ + tst_bitmaps_pthread \ ext2_tdbtool mkjournal debug_cmds.c tst_cmds.c extent_cmds.c \ ../libext2fs.a ../libext2fs_p.a ../libext2fs_chk.a \ crc32c_table.h gen_crc32ctable tst_crc32c tst_libext2fs \
@@ -1169,6 +1178,12 @@ tst_bitmaps_standalone.o: $(srcdir)/tst_bitmaps_standalone.c $(top_builddir)/lib $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h +tst_bitmaps_pthread.o: $(srcdir)/tst_bitmaps_pthread.c $(top_builddir)/lib/config.h \ + $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ + $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \ + $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ + $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ + $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h undo_io.o: $(srcdir)/undo_io.c $(top_builddir)/lib/config.h \ $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \
lib/ext2fs/tst_bitmaps_pthread.c new file +247 -0
247 lines not on this page · raw message
Signature · 1 line
2.35.3
RFC 9/13 badblocks: Add badblocks merge logic
In reply to Ritesh HarjaniFrom: Wang Shilong <[email protected]>
Add badblocks merge logic
--- lib/ext2fs/badblocks.c | 75 ++++++++++++++++++++++++++++++++++++++++++ lib/ext2fs/ext2fs.h | 2 ++ 2 files changed, 77 insertions(+)
lib/ext2fs/badblocks.c +75 -0
@@ -56,6 +56,74 @@ static errcode_t make_u32_list(int size, int num, __u32 *list,return 0 ; } +static inline int insert_ok (blk_t *array,int cnt,blk_t new) +{ +return (cnt ==0 || array[cnt -1 ] != new); +} + +/* +* Merge list from src to dest +*/ +static errcode_t merge_u32_list (ext2_u32_list src,ext2_u32_list dest) +{ +errcode_t retval; +int src_count = src->num; +int dest_count = dest->num; +int size = src_count + dest_count; +int size_entry =sizeof (blk_t); +blk_t *array; +blk_t *src_array = src->list; +blk_t *dest_array = dest->list; +int src_index =0 ; +int dest_index =0 ; +int uniq_cnt =0 ; + +if (src->num ==0 ) +return 0 ; + + retval =ext2fs_get_array (size, size_entry, &array); +if (retval) +return retval; + +/* +* It is possible that src list and dest list could be +* duplicated when merging badblocks. +*/ +while (src_index < src_count || dest_index < dest_count) { +if (src_index >= src_count) { +for (; dest_index < dest_count; dest_index++) +if (insert_ok (array, uniq_cnt, dest_array[dest_index])) + array[uniq_cnt++] = dest_array[dest_index]; +break ; + } +if (dest_index >= dest_count) { +for (; src_index < src_count; src_index++) +if (insert_ok (array, uniq_cnt, src_array[src_index])) + array[uniq_cnt++] = src_array[src_index]; +break ; + } +if (src_array[src_index] < dest_array[dest_index]) { +if (insert_ok (array, uniq_cnt, src_array[src_index])) + array[uniq_cnt++] = src_array[src_index]; + src_index++; + }else if (src_array[src_index] > dest_array[dest_index]) { +if (insert_ok (array, uniq_cnt, dest_array[dest_index])) + array[uniq_cnt++] = dest_array[dest_index]; + dest_index++; + }else { +if (insert_ok (array, uniq_cnt, dest_array[dest_index])) + array[uniq_cnt++] = dest_array[dest_index]; + src_index++; + dest_index++; + } + } + +ext2fs_free_mem (&dest->list); + dest->list = array; + dest->num = uniq_cnt; + dest->size = size; +return 0 ; +}/* * This procedure creates an empty u32 list.
@@ -91,6 +159,13 @@ errcode_t ext2fs_badblocks_copy(ext2_badblocks_list src, (ext2_u32_list *) dest); } +errcode_t ext2fs_badblocks_merge (ext2_badblocks_list src, +ext2_badblocks_list dest) +{ +return merge_u32_list ((ext2_u32_list ) src, + (ext2_u32_list ) dest); +} +/* * This procedure frees a badblocks list. *
lib/ext2fs/ext2fs.h +2 -0
@@ -844,6 +844,8 @@ extern int ext2fs_badblocks_list_iterate(ext2_badblocks_iterate iter,extern void ext2fs_badblocks_list_iterate_end (ext2_badblocks_iterate iter);extern errcode_t ext2fs_badblocks_copy (ext2_badblocks_list src,ext2_badblocks_list *dest); +extern errcode_t ext2fs_badblocks_merge (ext2_badblocks_list src, +ext2_badblocks_list dest);extern int ext2fs_badblocks_equal (ext2_badblocks_list bb1,ext2_badblocks_list bb2);extern int ext2fs_u32_list_count (ext2_u32_list bb);
Signature · 1 line
2.35.3
RFC 10/13 tst_badblocks: Add unit test to verify badblocks list merge api
In reply to Ritesh HarjaniAdd unit test to verify badblocks list merge api i.e. ext2fs_badblocks_merge()
--- lib/ext2fs/tst_badblocks.c | 61 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-)
lib/ext2fs/tst_badblocks.c +59 -2
@@ -119,6 +119,40 @@ static void print_list(badblocks_list bb, int verify) } } +static void do_list_merge_verify (badblocks_list bb,badblocks_list bbm,int verify) +{ +errcode_t retval; +badblocks_iterate iter; +blk_t blk; +int i, ok; + + retval =ext2fs_badblocks_merge (bb, bbm); +if (retval) { +com_err ("do_list_merge_verify" , retval,"while doing list merge" ); +return ; + } + +if (!verify) +return ; + + retval =ext2fs_badblocks_list_iterate_begin (bb, &iter); +if (retval) { +com_err ("do_list_merge_verify" , retval,"while setting up iterator" ); +return ; + } + +while (ext2fs_badblocks_list_iterate (iter, &blk)) { + retval =ext2fs_badblocks_list_test (bbm, blk); +if (retval ==0 ) { +printf (" --- NOT OK\n" ); + test_fail++; +return ; + } + } +ext2fs_badblocks_list_iterate_end (iter); +printf (" --- OK\n" ); +} +static void validate_test_seq (badblocks_list bb,blk_t *vec) {int i, match, ok;
@@ -275,13 +309,13 @@ out:int main (int argc,char **argv) { -badblocks_list bb1, bb2, bb3, bb4, bb5; +badblocks_list bb1, bb2, bb3, bb4, bb5, bbm;int equal;errcode_t retval;add_error_table (&et_ext2_error_table); - bb1 = bb2 = bb3 = bb4 = bb5 =0 ; + bb1 = bb2 = bb3 = bb4 = bb5 = bbm =0 ;printf ("test1: " ); retval =create_test_list (test1, &bb1);
@@ -346,6 +380,27 @@ int main(int argc, char **argv)printf ("\n" ); } +printf ("Create merge bb list\n" ); + retval =ext2fs_badblocks_list_create (&bbm,5 ); +if (retval) { +com_err ("ext2fs_badblocks_list_create" , retval,"while creating list" ); + test_fail++; + } + +printf ("Merge & Verify all bb{1..5} into bbm\n" ); +if (bb1 && bb2 && bb3 && bb4 && bb5 && bbm) { +printf ("Merge bb1 into bbm" ); +do_list_merge_verify (bb1, bbm,1 ); +printf ("Merge bb2 into bbm" ); +do_list_merge_verify (bb2, bbm,1 ); +printf ("Merge bb3 into bbm" ); +do_list_merge_verify (bb3, bbm,1 ); +printf ("Merge bb4 into bbm" ); +do_list_merge_verify (bb4, bbm,1 ); +printf ("Merge bb5 into bbm" ); +do_list_merge_verify (bb5, bbm,1 ); + } +file_test (bb4);file_test_invalid (bb4);
@@ -363,6 +418,8 @@ int main(int argc, char **argv)ext2fs_badblocks_list_free (bb4);if (bb5)ext2fs_badblocks_list_free (bb5); +if (bbm) +ext2fs_badblocks_list_free (bbm);return test_fail;
Signature · 1 line
2.35.3
RFC 11/13 dblist: add dblist merge logic
In reply to Ritesh HarjaniFrom: Li Xi <[email protected]>
This adds dblist merge logic.
TODO: Add a unit test for core operations of dblist. Currently there is no such test for this.
--- lib/ext2fs/dblist.c | 36 ++++++++++++++++++++++++++++++++++++ lib/ext2fs/ext2fs.h | 1 + 2 files changed, 37 insertions(+)
lib/ext2fs/dblist.c +36 -0
@@ -119,6 +119,42 @@ errcode_t ext2fs_copy_dblist(ext2_dblist src, ext2_dblist *dest)return 0 ; } +/* +* Merge a directory block list @src to @dest +*/ +errcode_t ext2fs_merge_dblist (ext2_dblist src,ext2_dblist dest) +{ +unsigned long long src_count = src->count; +unsigned long long dest_count = dest->count; +unsigned long long size = src_count + dest_count; +size_t size_entry =sizeof (struct ext2_db_entry2 ); +struct ext2_db_entry2 *array, *array2; +errcode_t retval; + +if (src_count ==0 ) +return 0 ; + +if (src->sorted || (dest->sorted && dest_count !=0 )) +return EINVAL ; + + retval =ext2fs_get_array (size, size_entry, &array); +if (retval) +return retval; + + array2 = array; +memcpy (array, src->list, src_count * size_entry); + array += src_count; +memcpy (array, dest->list, dest_count * size_entry); +ext2fs_free_mem (&dest->list); + + dest->list = array2; + dest->count = src_count + dest_count; + dest->size = size; + dest->sorted =0 ; + +return 0 ; +} +/* * Close a directory block list *
lib/ext2fs/ext2fs.h +1 -0
@@ -1146,6 +1146,7 @@ extern errcode_t ext2fs_add_dir_block(ext2_dblist dblist, ext2_ino_t ino, blk_t blk, int blockcnt);extern errcode_t ext2fs_add_dir_block2 (ext2_dblist dblist,ext2_ino_t ino,blk64_t blk,e2_blkcnt_t blockcnt); +extern errcode_t ext2fs_merge_dblist (ext2_dblist src,ext2_dblist dest);extern void ext2fs_dblist_sort (ext2_dblist dblist,EXT2_QSORT_TYPE (*sortfunc)(const void *,const void *));
Signature · 1 line
2.35.3
RFC 12/13 libext2fs: dupfs: Add fs clone & merge api
In reply to Ritesh HarjaniFrom: Saranya Muruganandam <[email protected]>
This patch mainly adds "parent" & "clone_flags" member in ext2_filsys struct for enabling multi-threading. Based on what CLONE flags will be passed from the client of libext2fs down to ext2fs_clone_fs(), those structures/bitmaps will be cloned (thread-aware child copy) and rest will be shared with the parent fs.
The same flags will also help to merge those cloned bitmap structures back into the parent bitmaps when ext2fs_merge_fs() will be called with childfs struct.
Review couple of todos within the patch.
1. I think we don't need refcount here.
2. For io_channel_close(), I think that might be required here (even though earlier I thought it should be done by the caller), before freeing childfs.
[added todos, modified naming, used #ifdef HAVE_PTHREAD, small bug fix in calling io_channel_close(), later added a test case against this patch]
--- lib/ext2fs/dupfs.c | 149 ++++++++++++++++++++++++++++++++++++++++++++ lib/ext2fs/ext2fs.h | 24 +++++++ 2 files changed, 173 insertions(+)
lib/ext2fs/dupfs.c +149 -0
@@ -14,8 +14,12 @@#if HAVE_UNISTD_H #include <unistd.h> #endif +#if HAVE_PTHREAD_H +#include <pthread.h> +#endif #include <time.h> #include <string.h> +#include <assert.h> #include "ext2_fs.h" #include "ext2fsP.h"
@@ -120,3 +124,148 @@ errout: } +#ifdef HAVE_PTHREAD +errcode_t ext2fs_clone_fs (ext2_filsys fs,ext2_filsys *dest,unsigned int flags) +{ +errcode_t retval; +ext2_filsys childfs; + +EXT2_CHECK_MAGIC (fs,EXT2_ET_MAGIC_EXT2FS_FILSYS ); + + retval =ext2fs_get_mem (sizeof (struct struct_ext2_filsys ), &childfs); +if (retval) +return retval; + +/* make an exact copy implying lists and memory structures are shared */ +memcpy (childfs, fs,sizeof (struct struct_ext2_filsys )); + childfs->inode_map =NULL ; + childfs->block_map =NULL ; + childfs->badblocks =NULL ; + childfs->dblist =NULL ; + +pthread_mutex_lock (&fs->refcount_mutex); + fs->refcount++; +pthread_mutex_unlock (&fs->refcount_mutex); + +if ((flags &EXT2FS_CLONE_INODE ) && fs->inode_map) { + retval =ext2fs_copy_bitmap (fs->inode_map, &childfs->inode_map); +if (retval) +return retval; + childfs->inode_map->fs = childfs; + } + +if ((flags &EXT2FS_CLONE_BLOCK ) && fs->block_map) { + retval =ext2fs_copy_bitmap (fs->block_map, &childfs->block_map); +if (retval) +return retval; + childfs->block_map->fs = childfs; + } + +if ((flags &EXT2FS_CLONE_BADBLOCKS ) && fs->badblocks) { + retval =ext2fs_badblocks_copy (fs->badblocks, &childfs->badblocks); +if (retval) +return retval; + } + +if ((flags &EXT2FS_CLONE_DBLIST ) && fs->dblist) { + retval =ext2fs_copy_dblist (fs->dblist, &childfs->dblist); +if (retval) +return retval; + childfs->dblist->fs = childfs; + } + +/* icache when NULL will be rebuilt if needed */ + childfs->icache =NULL ; + + childfs->clone_flags = flags; + childfs->parent = fs; + *dest = childfs; + +return 0 ; +} + +errcode_t ext2fs_merge_fs (ext2_filsys fs) +{ +errcode_t retval =0 ; +ext2_filsys dest = fs->parent; +ext2_filsys src = fs; +unsigned int flags = fs->clone_flags; + +pthread_mutex_lock (&fs->refcount_mutex); + fs->refcount--; +assert (fs->refcount >=0 ); +pthread_mutex_unlock (&fs->refcount_mutex); + +if ((flags &EXT2FS_CLONE_INODE ) && src->inode_map) { +if (dest->inode_map ==NULL ) { + dest->inode_map = src->inode_map; + src->inode_map =NULL ; + }else { + retval =ext2fs_merge_bitmap (src->inode_map, dest->inode_map,NULL ,NULL ); +if (retval) + goto out; + } + dest->inode_map->fs = dest; + } + +if ((flags &EXT2FS_CLONE_BLOCK ) && src->block_map) { +if (dest->block_map ==NULL ) { + dest->block_map = src->block_map; + src->block_map =NULL ; + }else { + retval =ext2fs_merge_bitmap (src->block_map, dest->block_map,NULL ,NULL ); +if (retval) + goto out; + } + dest->block_map->fs = dest; + } + +if ((flags &EXT2FS_CLONE_BADBLOCKS ) && src->badblocks) { +if (dest->badblocks ==NULL ) + retval =ext2fs_badblocks_copy (src->badblocks, &dest->badblocks); +else + retval =ext2fs_badblocks_merge (src->badblocks, dest->badblocks); +if (retval) + goto out; + } + +if ((flags &EXT2FS_CLONE_DBLIST ) && src->dblist) { +if (dest->dblist ==NULL ) { + dest->dblist = src->dblist; + src->dblist =NULL ; + }else { + retval =ext2fs_merge_dblist (src->dblist, dest->dblist); +if (retval) + goto out; + } + dest->dblist->fs = dest; + } + + dest->flags |= src->flags; +if (!(dest->flags &EXT2_FLAG_VALID )) +ext2fs_unmark_valid (dest); + +if (src->icache) { +ext2fs_free_inode_cache (src->icache); + src->icache =NULL ; + } + +out: +/* TODO check if io_channel_close is called correctly here? */ +if (src->io) +io_channel_close (src->io); + +if ((flags &EXT2FS_CLONE_INODE ) && src->inode_map) +ext2fs_free_generic_bmap (src->inode_map); +if ((flags &EXT2FS_CLONE_BLOCK ) && src->block_map) +ext2fs_free_generic_bmap (src->block_map); +if ((flags &EXT2FS_CLONE_BADBLOCKS ) && src->badblocks) +ext2fs_badblocks_list_free (src->badblocks); +if ((flags &EXT2FS_CLONE_DBLIST ) && src->dblist) +ext2fs_free_dblist (src->dblist); + +ext2fs_free_mem (&src); + +return retval; +} +#endif
lib/ext2fs/ext2fs.h +24 -0
@@ -12,6 +12,10 @@#ifndef _EXT2FS_EXT2FS_H#define _EXT2FS_EXT2FS_H +#ifdef HAVE_PTHREAD_H +#include <pthread.h> +#endif +#ifdef __GNUC__#define EXT2FS_ATTR (x) __attribute__(x)#else
@@ -330,6 +334,14 @@ struct struct_ext2_filsys {struct ext2fs_hashmap * block_sha_map;const struct ext2fs_nls_table *encoding; + +#ifdef HAVE_PTHREAD +struct struct_ext2_filsys *parent; +/* TODO do we need refcount? */ +size_t refcount; +pthread_mutex_t refcount_mutex; +unsigned int clone_flags; +#endif };#if EXT2_FLAT_INCLUDES
@@ -1056,6 +1068,18 @@ extern errcode_t ext2fs_move_blocks(ext2_filsys fs,/* check_desc.c */ extern errcode_t ext2fs_check_desc (ext2_filsys fs); +#ifdef HAVE_PTHREAD +/* flags for ext2fs_clone_fs */ +#define EXT2FS_CLONE_BLOCK 0x0001 +#define EXT2FS_CLONE_INODE 0x0002 +#define EXT2FS_CLONE_BADBLOCKS 0x0004 +#define EXT2FS_CLONE_DBLIST 0x0008 + +extern errcode_t ext2fs_clone_fs (ext2_filsys fs,ext2_filsys *dest, +unsigned int flags); +extern errcode_t ext2fs_merge_fs (ext2_filsys fs); +#endif +/* closefs.c */ extern errcode_t ext2fs_close (ext2_filsys fs);extern errcode_t ext2fs_close2 (ext2_filsys fs,int flags);
--
2.35.3
RFC 13/13 tst_libext2fs_pthread: Add libext2fs merge/clone unit tests
In reply to Ritesh HarjaniThis adds a unit tests for libext2fs merge/clone apis and uses pthreads to test the functionality correctly.
TODO:
We can also add EXT2FS_CLONE_BADBLOCKS and EXT2FS_CLONE_DBLIST test as well into it.
--- lib/ext2fs/Makefile.in | 17 +- lib/ext2fs/tst_libext2fs_pthread.c | 322 +++++++++++++++++++++++++++++ 2 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 lib/ext2fs/tst_libext2fs_pthread.c
lib/ext2fs/Makefile.in +15 -2
@@ -229,6 +229,7 @@ SRCS= ext2_err.c \ $(srcdir)/tst_libext2fs.c \ $(srcdir)/tst_bitmaps_standalone.c \ $(srcdir)/tst_bitmaps_pthread.c \ + $(srcdir)/tst_libext2fs_pthread.c \ $(DEBUG_SRCS) HFILES= bitops.h ext2fs.h ext2_io.h ext2_fs.h ext2_ext_attr.h ext3_extents.h \
@@ -374,6 +375,11 @@ tst_bitmaps_pthread: tst_bitmaps_pthread.o $(STATIC_LIBEXT2FS) $(DEPSTATIC_LIBCO $(Q) $(CC) -o tst_bitmaps_pthread tst_bitmaps_pthread.o $(ALL_LDFLAGS) \ $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) $(SYSLIBS) +tst_libext2fs_pthread: tst_libext2fs_pthread.o $(STATIC_LIBEXT2FS) $(DEPSTATIC_LIBCOM_ERR) + $(E) " LD $@" + $(Q) $(CC) -o tst_libext2fs_pthread tst_libext2fs_pthread.o $(ALL_LDFLAGS) \ + $(STATIC_LIBEXT2FS) $(STATIC_LIBCOM_ERR) $(SYSLIBS) + ext2_tdbtool: tdbtool.o $(E) " LD $@" $(Q) $(CC) -o ext2_tdbtool tdbtool.o tdb.o $(ALL_LDFLAGS) $(SYSLIBS)
@@ -546,7 +552,7 @@ fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ tst_super_size tst_types tst_inode_size tst_csum tst_crc32c tst_bitmaps \ tst_inline tst_inline_data tst_libext2fs tst_sha256 tst_sha512 \ tst_digest_encode tst_getsize tst_getsectsize tst_bitmaps_standalone \ - tst_bitmaps_pthread + tst_bitmaps_pthread tst_libext2fs_pthread $(TESTENV) ./tst_bitops $(TESTENV) ./tst_badblocks $(TESTENV) ./tst_iscan
@@ -571,6 +577,7 @@ fullcheck check:: tst_bitops tst_badblocks tst_iscan tst_types tst_icount \ $(TESTENV) ./tst_digest_encode $(TESTENV) ./tst_bitmaps_standalone $(TESTENV) ./tst_bitmaps_pthread + $(TESTENV) ./tst_libext2fs_pthread installdirs:: $(E) " MKDIR_P $(libdir) $(includedir)/ext2fs"
@@ -606,7 +613,7 @@ clean:: tst_bitmaps tst_bitmaps_out tst_extents tst_inline \ tst_inline_data tst_inode_size tst_bitmaps_cmd.c \ tst_digest_encode tst_sha256 tst_sha512 tst_bitmaps_standalone \ - tst_bitmaps_pthread \ + tst_bitmaps_pthread tst_libext2fs_pthread \ ext2_tdbtool mkjournal debug_cmds.c tst_cmds.c extent_cmds.c \ ../libext2fs.a ../libext2fs_p.a ../libext2fs_chk.a \ crc32c_table.h gen_crc32ctable tst_crc32c tst_libext2fs \
@@ -1184,6 +1191,12 @@ tst_bitmaps_pthread.o: $(srcdir)/tst_bitmaps_pthread.c $(top_builddir)/lib/confi $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h +tst_libext2fs_pthread.o: $(srcdir)/tst_libext2fs_pthread.c $(top_builddir)/lib/config.h \ + $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ + $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \ + $(srcdir)/ext2_fs.h $(srcdir)/ext3_extents.h $(top_srcdir)/lib/et/com_err.h \ + $(srcdir)/ext2_io.h $(top_builddir)/lib/ext2fs/ext2_err.h \ + $(srcdir)/ext2_ext_attr.h $(srcdir)/hashmap.h $(srcdir)/bitops.h undo_io.o: $(srcdir)/undo_io.c $(top_builddir)/lib/config.h \ $(top_builddir)/lib/dirpaths.h $(srcdir)/ext2_fs.h \ $(top_builddir)/lib/ext2fs/ext2_types.h $(srcdir)/ext2fs.h \
lib/ext2fs/tst_libext2fs_pthread.c new file +322 -0
322 lines not on this page · raw message
Signature · 1 line
2.35.3