Add threading support to e2fsprogs
cover Add threading support to e2fsprogs
This patch set adds the infrastructure to support threading to libext2fs. It makes the unix_io I/O Manager thread-aware. Wang's parallel bitmap code has been adapted to use the new threading infrastructure.
The code has been tested with TSAN and ASAN built into gcc 10.2:
configure 'CFLAGS=-g -fsanitize=thread' 'LDFLAGS=-fsanitize=thread' make clean ; make -j16 ; make -j16 check
configure 'CFLAGS=-g -fsanitize=address' 'LDFLAGS=-fsanitize=address' make clean ; make -j16 ; make -j16 check
As I needed to excerpt out some of the changes to generated patches in "Add configure and build support for the pthreads", the full patch series can be found in git:
git fetch https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git pthreads
Theodore Ts'o (4):
Add configure and build support for the pthreads library
libext2fs: add threading support to the I/O manager abstraction
libext2fs: allow the unix_io manager's cache to be disabled and
re-enabled
Enable threaded support for e2fsprogs' applications.
Wang Shilong (1):
ext2fs: parallel bitmap loading
MCONFIG.in | 12 +- aclocal.m4 | 486 ++++++++++++++++++++++++ configure | 814 ++++++++++++++++++++++++++++++++++++---- configure.ac | 24 ++ debugfs/debugfs.c | 6 +- e2fsck/unix.c | 2 +- lib/config.h.in | 351 +---------------- lib/ext2fs/ext2_io.h | 3 + lib/ext2fs/ext2fs.h | 9 + lib/ext2fs/openfs.c | 2 + lib/ext2fs/rw_bitmaps.c | 323 +++++++++++++--- lib/ext2fs/test_io.c | 6 +- lib/ext2fs/undo_io.c | 2 + lib/ext2fs/unix_io.c | 156 +++++++- misc/dumpe2fs.c | 2 +- misc/e2freefrag.c | 2 +- misc/e2fuzz.c | 4 +- misc/e2image.c | 3 +- misc/fuse2fs.c | 3 +- misc/tune2fs.c | 3 +- resize/main.c | 2 +- 21 files changed, 1719 insertions(+), 496 deletions(-)
Signature · 1 line
2.28.0
RFC PATCH 1/5 Add configure and build support for the pthreads library
Support for pthreads can be forcibly disabled by passing "--without-pthread" to the configure script.
The actual changes in this commit are in configure.ac and MCONFIG.in; the other files were generated as a result of running aclocal, autoconf, and autoheader on a Debian testing system.
Note: the autoconf-archive package must now be installed before rerunning aclocal, to supply the AX_PTHREAD macro.
---
[ I've removed the diffs of the generated files since they would make
the diff too large for the vger mailing lists. --Ted ]
MCONFIG.in | 12 +- aclocal.m4 | 486 +++++++++++++++++++++++++++++ configure | 814 +++++++++++++++++++++++++++++++++++++++++++----- configure.ac | 24 ++ lib/config.h.in | 351 +-------------------- 5 files changed, 1271 insertions(+), 416 deletions(-)
MCONFIG.in +7 -5
@@ -86,15 +86,17 @@ SYSTEMD_SYSTEM_UNIT_DIR = @systemd_system_unit_dir@ SANITIZER_CFLAGS = @lto_cflags@ @ubsan_cflags@ @addrsan_cflags@ @threadsan_cflags@ SANITIZER_LDFLAGS = @lto_ldflags@ @ubsan_ldflags@ @addrsan_ldflags@ @threadsan_ldflags@ -CC = @CC@ +CC = @PTHREAD_CC@ BUILD_CC = @BUILD_CC@ +PTHREAD_CFLAGS = @PTHREAD_CFLAGS@ +PTHREAD_LIBS = @PTHREAD_LIBS@ CFLAGS = @CFLAGS@ CFLAGS_SHLIB = @CFLAGS_SHLIB@ CFLAGS_STLIB = @CFLAGS_STLIB@ CPPFLAGS = @INCLUDES@ -ALL_CFLAGS = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) -ALL_CFLAGS_SHLIB = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS_SHLIB) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) -ALL_CFLAGS_STLIB = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS_STLIB) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) +ALL_CFLAGS = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS) $(PTHREAD_CFLAGS) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) +ALL_CFLAGS_SHLIB = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS_SHLIB) $(PTHREAD_CFLAGS) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) +ALL_CFLAGS_STLIB = $(CPPFLAGS) $(SANITIZER_CFLAGS) $(CFLAGS_STLIB) $(PTHREAD_CFLAGS) $(CFLAGS_WARN) @DEFS@ $(LOCAL_CFLAGS) LDFLAGS = $(SANITIZER_LDFLAGS) @LDFLAGS@ LDFLAGS_SHLIB = $(SANITIZER_LDFLAGS) @LDFLAGS_SHLIB@ ALL_LDFLAGS = $(LDFLAGS) @LDFLAG_DYNAMIC@
@@ -139,7 +141,7 @@ LIBFUSE = @FUSE_LIB@ LIBSUPPORT = $(LIBINTL) $(LIB)/libsupport@STATIC_LIB_EXT@ LIBBLKID = @LIBBLKID@ @PRIVATE_LIBS_CMT@ $(LIBUUID) LIBINTL = @LIBINTL@ -SYSLIBS = @LIBS@ +SYSLIBS = @LIBS@ @PTHREAD_LIBS@ DEPLIBSS = $(LIB)/libss@LIB_EXT@ DEPLIBCOM_ERR = $(LIB)/libcom_err@LIB_EXT@ DEPLIBUUID = @DEPLIBUUID@
configure.ac +24 -0
@@ -773,6 +773,30 @@ fi dnl dnl dnl +AC_ARG_WITH([pthread], +[ --without-pthread disable use of pthread support], +[if test "$withval" = "no" +then + try_pthread="" + AC_MSG_RESULT([Disabling pthread support]) +else + try_pthread="yes" + AC_MSG_RESULT([Testing for pthread support]) +fi] +, +try_pthread="yes" +AC_MSG_RESULT([Try testing for pthread support by default]) +) +if test "$try_pthread" = "yes" +then +AX_PTHREAD +else +test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" +AC_SUBST([PTHREAD_CC]) +fi +dnl +dnl +dnl AH_TEMPLATE([USE_UUIDD], [Define to 1 to build uuidd]) AC_ARG_ENABLE([uuidd], [ --disable-uuidd disable building the uuid daemon],
Signature · 1 line
2.28.0
RFC PATCH 2/5 libext2fs: add threading support to the I/O manager abstraction
In reply to Theodore Ts'oAdd initial implementation support for the unix_io manager. Applications which want to use threading should pass in IO_FLAG_THREADS when opening the channel. Channels which support threading (which as of this commit is unix_io and test_io if the backing io_manager supports threading) will set the CHANNEL_FLAGS_THREADS bit in io->flags. Library code or applications can test if threading is enabled by checking this flag.
Applications using libext2fs can pass in EXT2_FLAG_THREADS to ext2fs_open() or ext2fs_open2() to request threading support.
--- lib/ext2fs/ext2_io.h | 2 + lib/ext2fs/ext2fs.h | 1 + lib/ext2fs/openfs.c | 2 + lib/ext2fs/test_io.c | 6 +- lib/ext2fs/undo_io.c | 2 + lib/ext2fs/unix_io.c | 137 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 133 insertions(+), 17 deletions(-)
lib/ext2fs/ext2_io.h +2 -0
@@ -33,6 +33,7 @@ typedef struct struct_io_stats *io_stats;#define CHANNEL_FLAGS_WRITETHROUGH 0x01#define CHANNEL_FLAGS_DISCARD_ZEROES 0x02#define CHANNEL_FLAGS_BLOCK_DEVICE 0x04 +#define CHANNEL_FLAGS_THREADS 0x08#define io_channel_discard_zeroes_data (i) (i->flags & CHANNEL_FLAGS_DISCARD_ZEROES)
@@ -104,6 +105,7 @@ struct struct_io_manager {#define IO_FLAG_EXCLUSIVE 0x0002#define IO_FLAG_DIRECT_IO 0x0004#define IO_FLAG_FORCE_BOUNCE 0x0008 +#define IO_FLAG_THREADS 0x0010/* * Convenience functions....
lib/ext2fs/ext2fs.h +1 -0
@@ -206,6 +206,7 @@ typedef struct ext2_file *ext2_file_t;#define EXT2_FLAG_IGNORE_SB_ERRORS 0x800000#define EXT2_FLAG_BBITMAP_TAIL_PROBLEM 0x1000000#define EXT2_FLAG_IBITMAP_TAIL_PROBLEM 0x2000000 +#define EXT2_FLAG_THREADS 0x4000000/* * Special flag in the ext2 inode i_flag field that means that this is
lib/ext2fs/openfs.c +2 -0
@@ -170,6 +170,8 @@ errcode_t ext2fs_open2(const char *name, const char *io_options, io_flags |=IO_FLAG_EXCLUSIVE ;if (flags &EXT2_FLAG_DIRECT_IO ) io_flags |=IO_FLAG_DIRECT_IO ; +if (flags &EXT2_FLAG_THREADS ) + io_flags |=IO_FLAG_THREADS ; retval = manager->open (fs->device_name, io_flags, &fs->io);if (retval) goto cleanup;
lib/ext2fs/test_io.c +5 -1
@@ -197,6 +197,7 @@ static errcode_t test_open(const char *name, int flags, io_channel *channel) io->read_error =0 ; io->write_error =0 ; io->refcount =1 ; + io->flags =0 ;memset (data,0 ,sizeof (struct test_private_data )); data->magic =EXT2_ET_MAGIC_TEST_IO_CHANNEL ;
@@ -237,8 +238,11 @@ static errcode_t test_open(const char *name, int flags, io_channel *channel)if ((value =safe_getenv ("TEST_IO_WRITE_ABORT" )) !=NULL ) data->write_abort_count =strtoul (value,NULL ,0 ); -if (data->real) +if (data->real) { io->align = data->real->align; +if (data->real->flags &CHANNEL_FLAGS_THREADS ) + io->flags |=CHANNEL_FLAGS_THREADS ; + } *channel = io;return 0 ;
lib/ext2fs/undo_io.c +2 -0
@@ -698,6 +698,8 @@ static errcode_t undo_open(const char *name, int flags, io_channel *channel)int undo_fd =-1 ;errcode_t retval; +/* We don't support multi-threading, at least for now */ + flags &= ~IO_FLAG_THREADS ;if (name ==0 )return EXT2_ET_BAD_DEVICE_NAME ; retval =ext2fs_get_mem (sizeof (struct struct_io_channel ), &io);
lib/ext2fs/unix_io.c +121 -16
326 lines not on this page · raw message
Signature · 1 line
2.28.0
Re: [PATCH RFC 2/5] libext2fs: add threading support to the I/O manager abstraction
Quoted 30 lines · Theodore Ts'o
Add initial implementation support for the
unix_iomanager. Applications which want to use threading should pass inIO_FLAG_THREADSwhen opening the channel. Channels which support threading (which as of this commit isunix_ioandtest_ioif the backingio_managersupports threading) will set theCHANNEL_FLAGS_THREADSbit inio->flags. Library code or applications can test if threading is enabled by checking this flag.Applications using libext2fs can pass in
EXT2_FLAG_THREADStoext2fs_open()orext2fs_open2()to request threading support.Signed-off-by Theodore Ts'o <[email protected]>---
lib/ext2fs/unix_io.c +0
-0@@ -232,8 +287,10 @@ static errcode_t raw_read_blk(io_channel channel,
bounce_read:while (size > 0) {lib/ext2fs/unix_io.c +1
-0+mutex_lock (data,BOUNCE_MTX );actual = read(data->dev, data->bounce, channel->block_size); if (actual != channel->block_size) {lib/ext2fs/unix_io.c +1
-0+mutex_unlock (data,BOUNCE_MTX );actual = really_read; buf -= really_read; size += really_read;
lib/ext2fs/unix_io.c bounce_read: · line 308
Quoted 6 lines
lib/ext2fs/unix_io.c +1
-0@@ -246,6 +303,7 @@ bounce_read: really_read += actual; size -= actual; buf += actual; +mutex_unlock (data,BOUNCE_MTX ); }return 0 ;
Do you know how often we get into the "bounce_read" IO path? It seems like locking around the read would kill parallelism, but this code path also looks like a fallback, but maybe 100% used for blocksize != PAGE_SIZE?
Quoted 15 lines
lib/ext2fs/unix_io.c +0
-0@@ -341,11 +401,13 @@ static errcode_t raw_write_blk(io_channel channel,*/
bounce_write:while (size > 0) {lib/ext2fs/unix_io.c +1
-0+mutex_lock (data,BOUNCE_MTX );if (size < channel->block_size) { actual = read(data->dev, data->bounce, channel->block_size); if (actual != channel->block_size) { if (actual < 0) {lib/ext2fs/unix_io.c +1
-0+mutex_unlock (data,BOUNCE_MTX );retval = errno; goto error_out; }
lib/ext2fs/unix_io.c bounce_write: · line 430
Quoted 7 lines
lib/ext2fs/unix_io.c +1
-0@@ -362,6 +424,7 @@ bounce_write: goto error_out; } actual =write (data->dev, data->bounce, channel->block_size); +mutex_unlock (data,BOUNCE_MTX );if (actual <0 ) { retval = errno; goto error_out;
Ditto.
Quoted 14 lines
lib/ext2fs/unix_io.c +10
-0@@ -703,6 +773,25 @@ static errcode_t unix_open_channel(const char *name, int fd, setrlimit(RLIMIT_FSIZE, &rlim); } } +#endif +#ifdef HAVE_PTHREAD +if (flags &IO_FLAG_THREADS ) { + io->flags |=CHANNEL_FLAGS_THREADS ; + retval =pthread_mutex_init (&data->cache_mutex,NULL ); +if (retval) + goto cleanup; + retval =pthread_mutex_init (&data->bounce_mutex,NULL ); +if (retval) { +pthread_mutex_destroy (&data->cache_mutex);
lib/ext2fs/unix_io.c static errcode_t unix_open_channel(const char *name, int fd, · line 795
Quoted 10 lines
lib/ext2fs/unix_io.c +9
-0+ goto cleanup; + } + retval =pthread_mutex_init (&data->stats_mutex,NULL ); +if (retval) { +pthread_mutex_destroy (&data->cache_mutex); +pthread_mutex_destroy (&data->bounce_mutex); + goto cleanup; + } + }#endif
At one point you talked about using dlopen() or similar to link in the pthread library only if it is actually needed? Or is the linkage of the pthread library avoided by these functions not being called unless IO_FLAG_THREADS is set?
Cheers, Andreas
Re: [PATCH RFC 2/5] libext2fs: add threading support to the I/O manager abstraction
Quoted 4 lines · Andreas Dilger
Do you know how often we get into the "
bounce_read" IO path? It seems like locking around the read would kill parallelism, but this code path also looks like a fallback, but maybe 100% used for blocksize !=PAGE_SIZE?
It should be extremely rare. It only happens when Direct I/O is requested (which is rare to begin with, although it looks like there are people who are playing with some out of tree patchets to force e2image to use Direct I/O?), and the offset or the size of I/O isn't aligned with the system's direct I/O alignment requirements. (See ext2fs_get_dio_alignment() in lib/ext2fs/getsectsize.c)
Quoted 4 lines
At one point you talked about using
dlopen()or similar to link in the pthread library only if it is actually needed? Or is the linkage of the pthread library avoided by these functions not being called unlessIO_FLAG_THREADSis set?
So what I'm doing is just not trying to call those functions unless threading is required (e.g., IO_FLAG_THREADS is set, which would imply that EXT2_FLAG_THREADS was passed to ext2fs_open()). This won't help if the application is using static linking, but if the application is compiled statically, dlopen(3) is not guaranteed to work in any case.
So it's not perfect, and there may be some ancient AIX machine for which this might be problematic. But for all modern OS's that are linking to want to compile e2fsprogs, it should work. And I don't have access to an AIX machine these days, and I don't work for IBM anymore, so.... ¯\_(ツ)_/¯
:-)
- Ted
RFC PATCH 3/5 libext2fs: allow the unix_io manager's cache to be disabled and re-enabled
In reply to Theodore Ts'o--- lib/ext2fs/ext2_io.h | 1 + lib/ext2fs/unix_io.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+)
lib/ext2fs/ext2_io.h +1 -0
@@ -106,6 +106,7 @@ struct struct_io_manager {#define IO_FLAG_DIRECT_IO 0x0004#define IO_FLAG_FORCE_BOUNCE 0x0008#define IO_FLAG_THREADS 0x0010 +#define IO_FLAG_NOCACHE 0x0020/* * Convenience functions....
lib/ext2fs/unix_io.c +19 -0
@@ -942,6 +942,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,#ifdef NO_IO_CACHE return raw_read_blk (channel, data, block, count, buf);#else +if (data->flags &IO_FLAG_NOCACHE ) +return raw_read_blk (channel, data, block, count, buf);/* * If we're doing an odd-sized read or a very large read, * flush out the cache and then do a direct read.
@@ -1032,6 +1034,8 @@ static errcode_t unix_write_blk64(io_channel channel, unsigned long long block,#ifdef NO_IO_CACHE return raw_write_blk (channel, data, block, count, buf);#else +if (data->flags &IO_FLAG_NOCACHE ) +return raw_write_blk (channel, data, block, count, buf);/* * If we're doing an odd-sized write or a very large write, * flush out the cache completely and then do a direct write.
@@ -1161,6 +1165,7 @@ static errcode_t unix_set_option(io_channel channel, const char *option, {struct unix_private_data *data;unsigned long long tmp; +errcode_t retval;char *end;EXT2_CHECK_MAGIC (channel,EXT2_ET_MAGIC_IO_CHANNEL );
@@ -1179,6 +1184,20 @@ static errcode_t unix_set_option(io_channel channel, const char *option,return EXT2_ET_INVALID_ARGUMENT ;return 0 ; } +if (!strcmp (option,"cache" )) { +if (!arg) +return EXT2_ET_INVALID_ARGUMENT ; +if (!strcmp (arg,"on" )) { + data->flags &= ~IO_FLAG_NOCACHE ; +return 0 ; + } +if (!strcmp (arg,"off" )) { + retval =flush_cached_blocks (channel, data,0 ); + data->flags |=IO_FLAG_NOCACHE ; +return retval; + } +return EXT2_ET_INVALID_ARGUMENT ; + }return EXT2_ET_INVALID_ARGUMENT ; }
Signature · 1 line
2.28.0
RFC PATCH 4/5 ext2fs: parallel bitmap loading
In reply to Theodore Ts'oFrom: Wang Shilong <[email protected]>
In our benchmarking for PiB size filesystem, pass5 takes 10446s to finish and 99.5% of time takes on reading bitmaps.
It makes sense to reading bitmaps using multiple threads, a quickly benchmark show 10446s to 626s with 64 threads.
[ This has all of many bug fixes for rw_bitmaps.c from the original
luster patch set collapsed into a single commit. In addition it has
the new ext2fs_rw_bitmaps() api proposed by Ted. ]
--- lib/ext2fs/ext2fs.h | 8 + lib/ext2fs/rw_bitmaps.c | 323 +++++++++++++++++++++++++++++++++------- 2 files changed, 279 insertions(+), 52 deletions(-)
lib/ext2fs/ext2fs.h +8 -0
@@ -688,6 +688,14 @@ struct ext2_xattr_handle;#define XATTR_ABORT 1#define XATTR_CHANGED 2 +/* +* flags for ext2fs_rw_bitmaps() +*/ +#define EXT2FS_BITMAPS_WRITE 0x0001 +#define EXT2FS_BITMAPS_BLOCK 0x0002 +#define EXT2FS_BITMAPS_INODE 0x0004 +#define EXT2FS_BITMAPS_VALID_FLAGS 0x0007 +/* * function prototypes */
lib/ext2fs/rw_bitmaps.c +271 -52
431 lines not on this page · raw message
Signature · 1 line
2.28.0
Re: [PATCH RFC 4/5] ext2fs: parallel bitmap loading
Quoted 9 lines · Theodore Ts'o
From: Wang Shilong <[email protected]>
In our benchmarking for PiB size filesystem, pass5 takes 10446s to finish and 99.5% of time takes on reading bitmaps.
It makes sense to reading bitmaps using multiple threads, a quickly benchmark show 10446s to 626s with 64 threads.
Quoted 7 lines
[ This has all of many bug fixes for
rw_bitmaps.cfrom the original
luster patch set collapsed into a single commit. In addition it has
the newext2fs_rw_bitmaps()api proposed by Ted. ]Signed-off-by Wang Shilong <[email protected]>Signed-off-by Saranya Muruganandam <[email protected]>Signed-off-by Theodore Ts'o <[email protected]>
The patch looks generally good. Unfortunately, I don't have a large system available to verify the performance at this time, but it looks close enough to the original code that I don't think there is much risk of breakage.
One potential cross-platform build issue below, and some cosmetic suggestions, but you could add:
Quoted 9 lines
lib/ext2fs/rw_bitmaps.c +1
-1@@ -329,12 +369,20 @@ static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) }if (!bitmap_tail_verify ((unsigned char *) block_bitmap, block_nbytes, fs->blocksize -1 )) - tail_flags |=EXT2_FLAG_BBITMAP_TAIL_PROBLEM ; + *tail_flags |=EXT2_FLAG_BBITMAP_TAIL_PROBLEM ; }else memset (block_bitmap,0 , block_nbytes); cnt = block_nbytes <<3 ;
lib/ext2fs/rw_bitmaps.c static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) · line 385
Quoted 10 lines
lib/ext2fs/rw_bitmaps.c +8
-0+#ifdef HAVE_PTHREAD +if (mutex) +pthread_mutex_lock (mutex); +#endif retval =ext2fs_set_block_bitmap_range2 (fs->block_map, blk_itr, cnt, block_bitmap); +#ifdef HAVE_PTHREAD +if (mutex) +pthread_mutex_unlock (mutex); +#endif
(style) It wouldn't be terrible to have wrappers around these functions instead of inline #ifdef in the few places they are used, like:
#ifdef HAVE_PTHREAD
static void unix_pthread_mutex_lock(pthread_mutex_t *mutex)
{
if (mutex)
pthread_mutex_lock(mutex);
}
static void unix_pthread_mutex_unlock(pthread_mutex_t *mutex)
{
if (mutex)
pthread_mutex_unlock(mutex);
}#else
#define unix_pthread_mutex_lock(mutex) do {} while (0) #define unix_pthread_mutex_unlock(mutex) do {} while (0) #endif
lib/ext2fs/rw_bitmaps.c static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, memset(inode_bitmap, 0, inode_nbytes); · line 419
Quoted 8 lines
lib/ext2fs/rw_bitmaps.c +4
-0@@ -365,63 +413,229 @@ static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, memset(inode_bitmap, 0, inode_nbytes); cnt = inode_nbytes <<3 ; +if (mutex) +pthread_mutex_lock (mutex); retval =ext2fs_set_inode_bitmap_range2 (fs->inode_map, ino_itr, cnt, inode_bitmap); +if (mutex) +pthread_mutex_unlock (mutex);
(minor) These two pthread calls need #ifdef HAVE_PTHREAD or use wrappers.
Quoted 9 lines
lib/ext2fs/rw_bitmaps.c +9
-0+errcode_t ext2fs_rw_bitmaps (ext2_filsys fs,int flags,int num_threads) +{ +#ifdef HAVE_PTHREAD +if (((fs->io->flags &CHANNEL_FLAGS_THREADS ) ==0 ) || + (num_threads ==1 ) || (fs->flags &EXT2_FLAG_IMAGE_FILE )) + goto fallback; + +if (num_threads <0 ) { +#if defined(HAVE_SYSCONF ) && defined(_SC_NPROCESSORS_CONF)
lib/ext2fs/rw_bitmaps.c static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) · line 550
Quoted 10 lines
lib/ext2fs/rw_bitmaps.c +10
-0+ num_threads =sysconf (_SC_NPROCESSORS_CONF); +#else +/* +* Guess for now; eventually we should probably define +* ext2fs_get_num_cpus() and teach it how to get this info on +* MacOS, FreeBSD, etc. +* ref: https://stackoverflow.com/questions/150355 +*/ + num_threads =4 ; +#endif
(style) this #endif wouldn't hurt to have a /* HAVE_SYSCONF */ comment, but currently isn't too far from the #ifdef though it looks like it may become further away and harder to track in the future.
Quoted 3 lines
lib/ext2fs/rw_bitmaps.c +3
-0+if (num_threads <=1 ) + goto fallback; + }
[snip]
lib/ext2fs/rw_bitmaps.c static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) · line 627
Quoted 5 lines
lib/ext2fs/rw_bitmaps.c +2
-0+/* XXX should save and restore cache setting */ +io_channel_set_options (fs->io,"cache=on" );return retval;lib/ext2fs/rw_bitmaps.c +2
-0+fallback: +#endif
(style) this would definitely benefit from a /* HAVE_PTHREAD */ comment, since it is so far from the original #ifdef.
Cheers, Andreas
Re: [PATCH RFC 4/5] ext2fs: parallel bitmap loading
Quoted 34 lines · Andreas Dilger
Quoted 9 lines
lib/ext2fs/rw_bitmaps.c +1
-1@@ -329,12 +369,20 @@ static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) }if (!bitmap_tail_verify ((unsigned char *) block_bitmap, block_nbytes, fs->blocksize -1 )) - tail_flags |=EXT2_FLAG_BBITMAP_TAIL_PROBLEM ; + *tail_flags |=EXT2_FLAG_BBITMAP_TAIL_PROBLEM ; }else memset (block_bitmap,0 , block_nbytes); cnt = block_nbytes <<3 ;lib/ext2fs/rw_bitmaps.c static errcode_t read_bitmaps(ext2_filsys fs, int do_inode, int do_block) · line 385
Quoted 10 lines
lib/ext2fs/rw_bitmaps.c +8
-0+#ifdef HAVE_PTHREAD +if (mutex) +pthread_mutex_lock (mutex); +#endif retval =ext2fs_set_block_bitmap_range2 (fs->block_map, blk_itr, cnt, block_bitmap); +#ifdef HAVE_PTHREAD +if (mutex) +pthread_mutex_unlock (mutex); +#endif (style) It wouldn't be terrible to have wrappers around these functions instead of inline #ifdef in the few places they are used, like:
#ifdef
HAVE_PTHREAD
static voidunix_pthread_mutex_lock(pthread_mutex_t *mutex){ if (mutex) pthread_mutex_lock(mutex); } static void unix_pthread_mutex_unlock(pthread_mutex_t *mutex) { if (mutex) pthread_mutex_unlock(mutex); }
Quoted 4 lines
#else
#defineunix_pthread_mutex_lock(mutex)do {} while (0) #defineunix_pthread_mutex_unlock(mutex)do {} while (0) #endif
We'd also need to have a typedef for mutex_t which is either pthreads_mutex_t if pthreads are available, or an int (or some other placeholder type) if it isn't.
I had tried to make sure that rw_bitmaps.c will correctly compile with HAVE_PTHREAD and HAVE_PTHREAD_H are undefined. It looks like I didn't quite get it completely working since there's at leasts one function signature where we have an unprotected use of pthread_mutex_t, so that's something we should check before finalizing the patch --- in addition to the unprotected use of pthread_mutex_{lock,unlock} that you pointed out.
- Ted
RFC PATCH 5/5 Enable threaded support for e2fsprogs' applications.
In reply to Theodore Ts'o--- debugfs/debugfs.c | 6 ++++-- e2fsck/unix.c | 2 +- misc/dumpe2fs.c | 2 +- misc/e2freefrag.c | 2 +- misc/e2fuzz.c | 4 ++-- misc/e2image.c | 3 ++- misc/fuse2fs.c | 3 ++- misc/tune2fs.c | 3 ++- resize/main.c | 2 +- 9 files changed, 16 insertions(+), 11 deletions(-)
debugfs/debugfs.c +4 -2
@@ -231,7 +231,8 @@ void do_open_filesys(int argc, char **argv, int sci_idx EXT2FS_ATTR((unused)),int catastrophic =0 ;blk64_t superblock =0 ;blk64_t blocksize =0 ; -int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS ; +int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS | +EXT2_FLAG_THREADS ;char *data_filename =0 ;char *undo_file =NULL ;
@@ -2532,7 +2533,8 @@ int main(int argc, char **argv)#endif "[-c]] [device]" ;int c; -int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS ; +int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES | +EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;char *request =0 ;int exit_status =0 ;char *cmd_file =0 ;
e2fsck/unix.c +1 -1
@@ -1474,7 +1474,7 @@ int main (int argc, char *argv[]) } ctx->superblock = ctx->use_superblock; - flags =EXT2_FLAG_SKIP_MMP ; + flags =EXT2_FLAG_SKIP_MMP |EXT2_FLAG_THREADS ; restart:#ifdef CONFIG_TESTIO_DEBUG if (getenv ("TEST_IO_FLAGS" ) ||getenv ("TEST_IO_BLOCK" )) {
misc/dumpe2fs.c +1 -1
@@ -665,7 +665,7 @@ int main (int argc, char ** argv) device_name = argv[optind++]; flags =EXT2_FLAG_JOURNAL_DEV_OK |EXT2_FLAG_SOFTSUPP_FEATURES | -EXT2_FLAG_64BITS ; +EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;if (force) flags |=EXT2_FLAG_FORCE ;if (image_dump)
misc/e2freefrag.c +1 -1
@@ -363,7 +363,7 @@ static void collect_info(ext2_filsys fs, struct chunk_info *chunk_info, FILE *f)static void open_device (char *device_name,ext2_filsys *fs) {int retval; -int flag =EXT2_FLAG_FORCE |EXT2_FLAG_64BITS ; +int flag =EXT2_FLAG_FORCE |EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ; retval =ext2fs_open (device_name, flag,0 ,0 , unix_io_manager, fs);if (retval) {
misc/e2fuzz.c +2 -2
@@ -201,8 +201,8 @@ static int process_fs(const char *fsname) }/* Ensure the fs is clean and does not have errors */ - ret =ext2fs_open (fsname,EXT2_FLAG_64BITS ,0 ,0 , unix_io_manager, - &fs); + ret =ext2fs_open (fsname,EXT2_FLAG_64BITS |EXT2_FLAG_THREADS , +0 ,0 , unix_io_manager, &fs);if (ret) {fprintf (stderr,"%s: failed to open filesystem.\n" , fsname);
misc/e2image.c +2 -1
@@ -1482,7 +1482,8 @@ int main (int argc, char ** argv)ext2_filsys fs;char *image_fn, offset_opt[64 ];struct ext2_qcow2_hdr *header =NULL ; -int open_flag =EXT2_FLAG_64BITS |EXT2_FLAG_IGNORE_CSUM_ERRORS ; +int open_flag =EXT2_FLAG_64BITS |EXT2_FLAG_THREADS | +EXT2_FLAG_IGNORE_CSUM_ERRORS ;int img_type =0 ;int flags =0 ;int mount_flags =0 ;
misc/fuse2fs.c +2 -1
@@ -3727,7 +3727,8 @@ int main(int argc, char *argv[])errcode_t err;char *logfile;char extra_args[BUFSIZ ]; -int ret =0 , flags =EXT2_FLAG_64BITS |EXT2_FLAG_EXCLUSIVE ; +int ret =0 ; +int flags =EXT2_FLAG_64BITS |EXT2_FLAG_THREADS |EXT2_FLAG_EXCLUSIVE ;memset (&fctx,0 ,sizeof (fctx)); fctx.magic =FUSE2FS_MAGIC ;
misc/tune2fs.c +2 -1
@@ -2950,7 +2950,8 @@ retry_open:if ((open_flag &EXT2_FLAG_RW ) ==0 || f_flag) open_flag |=EXT2_FLAG_SKIP_MMP ; - open_flag |=EXT2_FLAG_64BITS |EXT2_FLAG_JOURNAL_DEV_OK ; + open_flag |=EXT2_FLAG_64BITS |EXT2_FLAG_THREADS | +EXT2_FLAG_JOURNAL_DEV_OK ;/* keep the filesystem struct around to dump MMP data */ open_flag |=EXT2_FLAG_NOFREE_ON_ERROR ;
resize/main.c +1 -1
@@ -402,7 +402,7 @@ int main (int argc, char ** argv)if (!(mount_flags &EXT2_MF_MOUNTED )) io_flags =EXT2_FLAG_RW |EXT2_FLAG_EXCLUSIVE ; - io_flags |=EXT2_FLAG_64BITS ; + io_flags |=EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;if (undo_file) { retval =resize2fs_setup_tdb (device_name, undo_file, &io_ptr);if (retval)
Signature · 1 line
2.28.0
Re: [PATCH RFC 5/5] Enable threaded support for e2fsprogs' applications.
Quoted 2 lines · Theodore Ts'o
Signed-off-by Theodore Ts'o <[email protected]>
My understanding is that as soon as the EXT2_FLAG_THREADS is added, and if the backend supports CHANNEL_FLAGS_THREADS, then the pthread code in the previous patch will "autothread" based on the number of CPUs in the system.
That will be nice for debugfs, which would otherwise take ages to start on a large filesystem if "-c" was not used (which also precludes any kind of modifications).
Quoted 146 lines
---
debugfs/debugfs.c | 6 ++++-- e2fsck/unix.c | 2 +- misc/dumpe2fs.c | 2 +- misc/e2freefrag.c | 2 +- misc/e2fuzz.c | 4 ++-- misc/e2image.c | 3 ++- misc/fuse2fs.c | 3 ++- misc/tune2fs.c | 3 ++- resize/main.c | 2 +- 9 files changed, 16 insertions(+), 11 deletions(-)debugfs/debugfs.c +4
-2@@ -231,7 +231,8 @@ void do_open_filesys(int argc, char **argv, int sci_idx EXT2FS_ATTR((unused)),int catastrophic =0 ;blk64_t superblock =0 ;blk64_t blocksize =0 ; -int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS ; +int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS | +EXT2_FLAG_THREADS ;char *data_filename =0 ;char *undo_file =NULL ;@@ -2532,7 +2533,8 @@ int main(int argc, char **argv)#endif "[-c]] [device]" ;int c; -int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES |EXT2_FLAG_64BITS ; +int open_flags =EXT2_FLAG_SOFTSUPP_FEATURES | +EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;char *request =0 ;int exit_status =0 ;char *cmd_file =0 ;e2fsck/unix.c +1
-1@@ -1474,7 +1474,7 @@ int main (int argc, char *argv[]) } ctx->superblock = ctx->use_superblock; - flags =EXT2_FLAG_SKIP_MMP ; + flags =EXT2_FLAG_SKIP_MMP |EXT2_FLAG_THREADS ;restart:
#ifdef CONFIG_TESTIO_DEBUG if (getenv("TEST_IO_FLAGS") || getenv("TEST_IO_BLOCK")) {misc/dumpe2fs.c +1
-1@@ -665,7 +665,7 @@ int main (int argc, char ** argv) device_name = argv[optind++]; flags =EXT2_FLAG_JOURNAL_DEV_OK |EXT2_FLAG_SOFTSUPP_FEATURES | -EXT2_FLAG_64BITS ; +EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;if (force) flags |=EXT2_FLAG_FORCE ;if (image_dump)misc/e2freefrag.c +1
-1@@ -363,7 +363,7 @@ static void collect_info(ext2_filsys fs, struct chunk_info *chunk_info, FILE *f)static void open_device (char *device_name,ext2_filsys *fs) {int retval; -int flag =EXT2_FLAG_FORCE |EXT2_FLAG_64BITS ; +int flag =EXT2_FLAG_FORCE |EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ; retval =ext2fs_open (device_name, flag,0 ,0 , unix_io_manager, fs);if (retval) {misc/e2fuzz.c +2
-2@@ -201,8 +201,8 @@ static int process_fs(const char *fsname) }/* Ensure the fs is clean and does not have errors */ - ret =ext2fs_open (fsname,EXT2_FLAG_64BITS ,0 ,0 , unix_io_manager, - &fs); + ret =ext2fs_open (fsname,EXT2_FLAG_64BITS |EXT2_FLAG_THREADS , +0 ,0 , unix_io_manager, &fs);if (ret) {fprintf (stderr,"%s: failed to open filesystem.\n" , fsname);misc/e2image.c +2
-1@@ -1482,7 +1482,8 @@ int main (int argc, char ** argv)ext2_filsys fs;char *image_fn, offset_opt[64 ];struct ext2_qcow2_hdr *header =NULL ; -int open_flag =EXT2_FLAG_64BITS |EXT2_FLAG_IGNORE_CSUM_ERRORS ; +int open_flag =EXT2_FLAG_64BITS |EXT2_FLAG_THREADS | +EXT2_FLAG_IGNORE_CSUM_ERRORS ;int img_type =0 ;int flags =0 ;int mount_flags =0 ;misc/fuse2fs.c +2
-1@@ -3727,7 +3727,8 @@ int main(int argc, char *argv[])errcode_t err;char *logfile;char extra_args[BUFSIZ ]; -int ret =0 , flags =EXT2_FLAG_64BITS |EXT2_FLAG_EXCLUSIVE ; +int ret =0 ; +int flags =EXT2_FLAG_64BITS |EXT2_FLAG_THREADS |EXT2_FLAG_EXCLUSIVE ;memset (&fctx,0 ,sizeof (fctx)); fctx.magic =FUSE2FS_MAGIC ;misc/tune2fs.c +2
-1@@ -2950,7 +2950,8 @@ retry_open:if ((open_flag &EXT2_FLAG_RW ) ==0 || f_flag) open_flag |=EXT2_FLAG_SKIP_MMP ; - open_flag |=EXT2_FLAG_64BITS |EXT2_FLAG_JOURNAL_DEV_OK ; + open_flag |=EXT2_FLAG_64BITS |EXT2_FLAG_THREADS | +EXT2_FLAG_JOURNAL_DEV_OK ;/* keep the filesystem struct around to dump MMP data */ open_flag |=EXT2_FLAG_NOFREE_ON_ERROR ;resize/main.c +1
-2@@ -402,7 +402,7 @@ int main (int argc, char ** argv)if (!(mount_flags &EXT2_MF_MOUNTED )) io_flags =EXT2_FLAG_RW |EXT2_FLAG_EXCLUSIVE ; - io_flags |=EXT2_FLAG_64BITS ; + io_flags |=EXT2_FLAG_64BITS |EXT2_FLAG_THREADS ;if (undo_file) { retval =resize2fs_setup_tdb (device_name, undo_file, &io_ptr);if (retval) --
Quoted 2 lines
2.28.0
Cheers, Andreas
Re: [PATCH RFC 5/5] Enable threaded support for e2fsprogs' applications.
Quoted 8 lines · Andreas Dilger
Quoted 2 lines · Theodore Ts'o
Signed-off-by Theodore Ts'o <[email protected]>My understanding is that as soon as the
EXT2_FLAG_THREADSis added, and if the backend supportsCHANNEL_FLAGS_THREADS, then the pthread code in the previous patch will "autothread" based on the number of CPUs in the system.
Yep.
Quoted 5 lines
That will be nice for debugfs, which would otherwise take ages to start on a large filesystem if "-c" was not used (which also precludes any kind of modifications).
Reviewed-by Andreas Dilger <[email protected]>
Yes, that's an issue that our SRE's have run across as well while debugging customer problems, which is one of the reasons why I've been interested in getting this change upstream first, even it takes a bit longer to get all of the parallel fsck changes reviewed and upstream.
- Ted