add progress indicator to btrfs-convert
Subsystems (3)
cover add progress indicator to btrfs-convert
Hi btrfs-ml,
I have tried a ext4->btrfs conversation of a 1TB (90%full) and have seen that btrfs has no progressbar or something like that. Furthermore I have found [1] a project idea for this.
I have tried a generic tasklet-based solution to print the status of the convert-process. Maybe it looks good to you than please add this to your repo. I have implemented this with the linux specific timerfd facility. To enable this you haver to use the '-p' option.
I have noticed that a conversation doesn't show all inodes as converted at the end. How can I improve this? A complete convert-process is showd here:
sfr@devhost ./btrfs-progs % ./btrfs-convert -p /dev/sde4 creating btrfs metadata.
copy inodes [o] [ 96/ 106]
creating ext2fs image file.
cleaning up system chunk.
conversion complete.
[1] https://btrfs.wiki.kernel.org/index.php/Project_ideas#btrfs-convert
Bye and thanks,
Silvio
Silvio Fricke (2):
btrfs-progs: add task-utils
btrfs-progs: convert: use task for progress indication of metadata
creation
Documentation/btrfs-convert.txt | 2 + Makefile | 6 +-- btrfs-convert.c | 60 ++++++++++++++++++++++-- task-util.c | 101 ++++++++++++++++++++++++++++++++++++++++ task-util.h | 32 +++++++++++++ 5 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 task-util.c create mode 100644 task-util.h
Signature · 1 line
2.1.2
PATCH 1/2 btrfs-progs: add task-utils
--- task-util.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ task-util.h | 33 +++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 task-util.c create mode 100644 task-util.h
task-util.c new file +121 -0
@@ -0,0 +1,121 @@ + +#include <pthread.h> +#include <sys/timerfd.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include "task-util.h" + +struct task_info *task_init (void *(*threadfn)(void *),int (*postfn)(void *), +void *thread_private) +{ +struct task_info *info =calloc (1 ,sizeof (struct task_info )); + +if (!info) +return NULL ; + + info->private_data = thread_private; + info->threadfn = threadfn; + info->postfn = postfn; + +return info; +} + +int task_start (struct task_info *info) +{ +int ret; + +if (!info) +return -1 ; + +if (!info->threadfn) +return -1 ; + + ret =pthread_create (&info->id,NULL , info->threadfn, + info->private_data); + +if (ret ==0 ) +pthread_detach (info->id); +else + info->id =-1 ; + +return ret; +} + +void task_stop (struct task_info *info) +{ +if (!info) +return ; + +if (info->periodic.timer_fd) +close (info->periodic.timer_fd); + +if (info->id >0 ) +pthread_cancel (info->id); + +if (info->postfn) + info->postfn (info->private_data); +} + +void task_deinit (struct task_info *info) +{ +if (!info) +return ; + +free (info); +} + +int task_period_start (struct task_info *info,unsigned int period_ms) +{ +unsigned int ns; +unsigned int sec; +struct itimerspec itval; + +if (!info) +return -1 ; + + info->periodic.timer_fd =timerfd_create (CLOCK_MONOTONIC ,0 ); +if (info->periodic.timer_fd ==-1 ) +return info->periodic.timer_fd; + + info->periodic.wakeups_missed =0 ; + + sec = period_ms/1000 ; + ns = (period_ms - (sec *1000 )) *1000 ; + itval.it_interval.tv_sec = sec; + itval.it_interval.tv_nsec = ns; + itval.it_value.tv_sec = sec; + itval.it_value.tv_nsec = ns; + +return timerfd_settime (info->periodic.timer_fd,0 , &itval,NULL ); +}; + +void task_period_wait (struct task_info *info) +{ +unsigned long long missed; +int ret; + +if (!info) +return ; + + ret =read (info->periodic.timer_fd, &missed,sizeof (missed)); +if (ret ==-1 ) { +perror ("read timer" ); +return ; + } + +if (missed >0 ) + info->periodic.wakeups_missed += (missed -1 ); +} + +void task_period_stop (struct task_info *info) +{ +if (!info) +return ; + +if (info->periodic.timer_fd) { +timerfd_settime (info->periodic.timer_fd,0 ,NULL ,NULL ); +close (info->periodic.timer_fd); + } +}
task-util.h new file +33 -0
@@ -0,0 +1,33 @@ + +#ifndef __PROGRESS_ +#define __PROGRESS_ + +#include <pthread.h> + +struct periodic_info { +int timer_fd; +unsigned long long wakeups_missed; +}; + +struct task_info { +struct periodic_info periodic; +pthread_t id; +void *private_data; +void *(*threadfn)(void *); +int (*postfn)(void *); +}; + +/* task life cycle */ +struct task_info *task_init (void *(*threadfn)(void *),int (*postfn)(void *), +void *thread_private); +int task_start (struct task_info *info); +void task_stop (struct task_info *info); +void task_deinit (struct task_info *info); + +/* periodic life cycle */ +int task_period_start (struct task_info *info,unsigned int period_ms); +void task_period_wait (struct task_info *info); +void task_period_stop (struct task_info *info); + +#endif /* __PROGRESS_ */ +
Signature · 1 line
2.1.2
PATCH 2/2 btrfs-progs: convert: use task for progress indication of metadata creation
In reply to Silvio Fricke--- Documentation/btrfs-convert.txt | 2 ++ Makefile | 6 ++-- btrfs-convert.c | 64 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 8 deletions(-)
Documentation/btrfs-convert.txt +2 -0
@@ -29,6 +29,8 @@ Roll back to ext2fs. set filesystem label during conversion. -L:: use label from the converted filesystem. +-p:: +Show progress of convertation. EXIT STATUS -----------
Makefile +3 -3
@@ -8,7 +8,7 @@ AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DBTRFS_FLAT_INCLUDES -fno-strict-alias CFLAGS = -g -O1 -fno-strict-aliasing -rdynamic objects = ctree.o disk-io.o radix-tree.o extent-tree.o print-tree.o \ root-tree.o dir-item.o file-item.o inode-item.o inode-map.o \ - extent-cache.o extent_io.o volumes.o utils.o repair.o \ + extent-cache.o extent_io.o volumes.o utils.o repair.o task-util.o \ qgroup.o raid6.o free-space-cache.o list_sort.o props.o \ ulist.o qgroup-verify.o backref.o cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
@@ -20,13 +20,13 @@ libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ uuid-tree.o utils-lib.o rbtree-utils.o libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \ crc32c.h list.h kerncompat.h radix-tree.h extent-cache.h \ - extent_io.h ioctl.h ctree.h btrfsck.h version.h + extent_io.h ioctl.h ctree.h btrfsck.h version.h task-util.h TESTS = fsck-tests.sh convert-tests.sh INSTALL = install prefix ?= /usr/local bindir = $(prefix)/bin -lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. +lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. -pthread libdir ?= $(prefix)/lib incdir = $(prefix)/include/btrfs LIBS = $(lib_LIBS) $(libs_static)
btrfs-convert.c +59 -5
132 lines not on this page · raw message
Signature · 1 line
2.1.2
Quoted 6 lines · Silvio Fricke
I have tried a
ext4->btrfsconversation of a 1TB (90%full) and have seen that btrfs has no progressbar or something like that. Furthermore I have found [1] a project idea for this.I have tried a generic tasklet-based solution to print the status of the convert-process. Maybe it looks good to you than please add this to your repo.
Added to integration for now to give it some more testing. There are potential users for the 'progress' feature (scrub, balance, replace) so we may build it on top of your patches though I haven't looked closer.
Re: [PATCH 1/2] btrfs-progs: add task-utils
In reply to Silvio FrickeHi,
so started reviewing the patches for inclusion in the 3.18 branch and found a few things that I've fixed locally, this is just FYI.
task-util.c
Quoted 13 lines · Silvio Fricke
Signed-off-by Silvio Fricke <[email protected]>--- task-util.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ task-util.h | 33 +++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 task-util.c create mode 100644 task-util.htask-util.c new file +0
-0
Renamed the file to task-utils.c
task-util.h line 3
Quoted 9 lines
task-util.h new file +3
-0@@ -0,0 +1,33 @@ + +#ifndef __PROGRESS_ +#define __PROGRESS_
__TASK_UTILS_H__
Quoted 19 lines
task-util.h new file +19
-0+ +#include <pthread.h> + +struct periodic_info { +int timer_fd; +unsigned long long wakeups_missed; +}; + +struct task_info { +struct periodic_info periodic; +pthread_t id; +void *private_data; +void *(*threadfn)(void *); +int (*postfn)(void *); +}; + +/* task life cycle */ +struct task_info *task_init (void *(*threadfn)(void *),int (*postfn)(void *), +void *thread_private);
task-util.h line 32
Quoted 10 lines
task-util.h new file +10
-0+int task_start (struct task_info *info); +void task_stop (struct task_info *info); +void task_deinit (struct task_info *info); + +/* periodic life cycle */ +int task_period_start (struct task_info *info,unsigned int period_ms); +void task_period_wait (struct task_info *info); +void task_period_stop (struct task_info *info); + +#endif /* __PROGRESS_ */
dtto
Re: [PATCH 2/2] btrfs-progs: convert: use task for progress indication of metadata creation
In reply to Silvio FrickeFixed locally, no need to resend the patch. JFYI the changes I made
Quoted 11 lines · Silvio Fricke
--- Documentation/btrfs-convert.txt | 2 ++ Makefile | 6 ++-- btrfs-convert.c | 64 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 8 deletions(-)Documentation/btrfs-convert.txt +0
-0
Documentation/btrfs-convert.txt Roll back to ext2fs. · line 33
Quoted 5 lines
Documentation/btrfs-convert.txt +2
-0@@ -29,6 +29,8 @@ Roll back to ext2fs. set filesystem label during conversion. -L:: use label from the converted filesystem. +-p:: +Show progress of convertation.
conversion
Quoted 16 lines
Documentation/btrfs-convert.txt +0
-0EXIT STATUS -----------Makefile +1
-1@@ -8,7 +8,7 @@ AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DBTRFS_FLAT_INCLUDES -fno-strict-alias CFLAGS = -g -O1 -fno-strict-aliasing -rdynamic objects = ctree.o disk-io.o radix-tree.o extent-tree.o print-tree.o \ root-tree.o dir-item.o file-item.o inode-item.o inode-map.o \ - extent-cache.o extent_io.o volumes.o utils.o repair.o \ + extent-cache.o extent_io.o volumes.o utils.o repair.o task-util.o \ qgroup.o raid6.o free-space-cache.o list_sort.o props.o \ ulist.o qgroup-verify.o backref.o cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
Makefile libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ · line 23
Quoted 5 lines
Makefile +1
-1@@ -20,13 +20,13 @@ libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ uuid-tree.o utils-lib.o rbtree-utils.o libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \ crc32c.h list.h kerncompat.h radix-tree.h extent-cache.h \ - extent_io.h ioctl.h ctree.h btrfsck.h version.h + extent_io.h ioctl.h ctree.h btrfsck.h version.h task-util.h
It's not used in library, removed
Makefile libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ · line 29
Quoted 7 lines
Makefile +1
-1TESTS = fsck-tests.sh convert-tests.sh INSTALL = install prefix ?= /usr/local bindir = $(prefix)/bin -lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. +lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. -pthread
dtto
Quoted 16 lines
Makefile +0
-0libdir ?= $(prefix)/lib incdir = $(prefix)/include/btrfs LIBS = $(lib_LIBS) $(libs_static)btrfs-convert.c +1
-0@@ -38,6 +38,7 @@#include "transaction.h" #include "crc32c.h" #include "utils.h" +#include "task-util.h" #include <ext2fs/ext2_fs.h> #include <ext2fs/ext2fs.h> #include <ext2fs/ext2_ext_attr.h>
btrfs-convert.c line 49
Quoted 4 lines
btrfs-convert.c +1
-0@@ -45,6 +46,41 @@#define INO_OFFSET (BTRFS_FIRST_FREE_OBJECTID - EXT2_ROOT_INO)#define EXT2_IMAGE_SUBVOL_OBJECTID BTRFS_FIRST_FREE_OBJECTID +struct private {
We'd still like to keep the namespace C++ clean, renamed to task_ctx
Quoted 64 lines
btrfs-convert.c +37
-2+uint32_t max_copy_inodes; +uint32_t cur_copy_inodes; +struct task_info *info; +}; + +static void *print_copied_inodes (void *p) +{ +struct private *priv = p; +const char work_indicator[] = {'.' ,'o' ,'O' ,'o' }; +uint32_t count =0 ; + +task_period_start (priv->info,1000 /* 1s */ ); +while (1 ) { + count++; +printf ("copy inodes [%c] [%10d/%10d]\r" , + work_indicator[count %4 ], priv->cur_copy_inodes, + priv->max_copy_inodes); +fflush (stdout); +task_period_wait (priv->info); + } + +return NULL ; +} + +static int after_copied_inodes (void *p) +{ +struct private *priv = p; + +printf ("\n" ); +task_period_stop (priv->info); + +return 0 ; +} +/* * Open Ext2fs in readonly mode, read block allocation bitmap and * inode bitmap into memory. @@ -1036,7 +1072,7 @@ fail:* scan ext2's inode bitmap and copy all used inodes. */ static int copy_inodes (struct btrfs_root *root,ext2_filsys ext2_fs, -int datacsum,int packing,int noxattr) +int datacsum,int packing,int noxattr,struct private *p) {int ret;errcode_t err;@@ -1068,6 +1104,7 @@ static int copy_inodes(struct btrfs_root *root, ext2_filsys ext2_fs, objectid, ext2_fs, ext2_ino, &ext2_inode, datacsum, packing, noxattr); + p->cur_copy_inodes++;if (ret)return ret;if (trans->blocks_used >=4096 ) {@@ -2197,7 +2234,7 @@ err: }static int do_convert (const char *devname,int datacsum,int packing,int noxattr, -int copylabel,const char *fslabel) +int copylabel,const char *fslabel,int progress) {int i, ret;int fd =-1 ;
btrfs-convert.c static int do_convert(const char *devname, int datacsum, int packing, int noxatt · line 2315
Quoted 5 lines
btrfs-convert.c +1
-1@@ -2275,11 +2312,23 @@ static int do_convert(const char *devname, int datacsum, int packing, int noxatt goto fail; }printf ("creating btrfs metadata.\n" ); - ret =copy_inodes (root, ext2_fs, datacsum, packing, noxattr); +struct private p = {
Declared at the beginning of the function, renamed from a single letter
Quoted 67 lines
btrfs-convert.c +19
-2+ .max_copy_inodes = (ext2_fs->super->s_inodes_count - ext2_fs->super->s_free_inodes_count), + .cur_copy_inodes = 0, + }; +if (progress) { + p.info =task_init (print_copied_inodes, after_copied_inodes, &p); +task_start (p.info); + } + ret =copy_inodes (root, ext2_fs, datacsum, packing, noxattr, &p);if (ret) {fprintf (stderr,"error during copy_inodes %d\n" , ret); goto fail; } +if (progress) { +task_stop (p.info); +task_deinit (p.info); + }printf ("creating ext2fs image file.\n" ); ext2_root =link_subvol (root,"ext2_saved" ,EXT2_IMAGE_SUBVOL_OBJECTID );if (!ext2_root) {@@ -2703,6 +2752,7 @@ static void print_usage(void)printf ("\t-r roll back to ext2fs\n" );printf ("\t-l LABEL set filesystem label\n" );printf ("\t-L use label from converted fs\n" ); +printf ("\t-p show converting progress\n" ); }int main (int argc,char *argv[])@@ -2714,11 +2764,12 @@ int main(int argc, char *argv[])int rollback =0 ;int copylabel =0 ;int usage_error =0 ; +int progress =0 ;char *file;char *fslabel =NULL ;while (1 ) { -int c =getopt (argc, argv,"dinrl:L" ); +int c =getopt (argc, argv,"dinrl:Lp" );if (c <0 )break ;switch (c) {@@ -2747,6 +2798,9 @@ int main(int argc, char *argv[])case 'L' : copylabel =1 ;break ; +case 'p' : + progress =1 ; +break ;default :print_usage ();return 1 ;@@ -2784,7 +2838,7 @@ int main(int argc, char *argv[])if (rollback) { ret =do_rollback (file); }else { - ret =do_convert (file, datacsum, packing, noxattr, copylabel, fslabel); + ret =do_convert (file, datacsum, packing, noxattr, copylabel, fslabel, progress); }if (ret)return 1 ;Signature · 6 lines
2.1.2 -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html