Skip to contentSkip to conversation

add progress indicator to btrfs-convert

PATCH In review Silvio Fricke 2 participants 6 messages Nov 9, 2014 – Nov 25, 2014

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover add progress indicator to btrfs-convert

Hi btrfs-ml,

I have tried a ext4->btrfs conversation of a 1TB (90%full) and have seen that btrfs has no progressbar or something like that. Furthermore I have found [1] a project idea for this.

I have tried a generic tasklet-based solution to print the status of the convert-process. Maybe it looks good to you than please add this to your repo. I have implemented this with the linux specific timerfd facility. To enable this you haver to use the '-p' option.

I have noticed that a conversation doesn't show all inodes as converted at the end. How can I improve this? A complete convert-process is showd here:

sfr@devhost ./btrfs-progs % ./btrfs-convert -p /dev/sde4 creating btrfs metadata.
copy inodes [o] [ 96/ 106]
creating ext2fs image file.
cleaning up system chunk.
conversion complete.

[1] https://btrfs.wiki.kernel.org/index.php/Project_ideas#btrfs-convert

Bye and thanks,
Silvio

Silvio Fricke (2):
btrfs-progs: add task-utils
btrfs-progs: convert: use task for progress indication of metadata
creation

 Documentation/btrfs-convert.txt |   2 +
 Makefile                        |   6 +--
 btrfs-convert.c                 |  60 ++++++++++++++++++++++--
 task-util.c                     | 101 ++++++++++++++++++++++++++++++++++++++++
 task-util.h                     |  32 +++++++++++++
 5 files changed, 193 insertions(+), 8 deletions(-)
 create mode 100644 task-util.c
 create mode 100644 task-util.h
Signature · 1 line
2.1.2
#1 lore raw

PATCH 1/2 btrfs-progs: add task-utils

---
 task-util.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 task-util.h |  33 +++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100644 task-util.c
 create mode 100644 task-util.h
task-util.c new file +121 -0
new file mode 100644
index 0000000..9268df7
@@ -0,0 +1,121 @@
+
+#include <pthread.h>
+#include <sys/timerfd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "task-util.h"
+
+struct task_info *task_init(void *(*threadfn)(void *), int (*postfn)(void *),
+			    void *thread_private)
+{
+	struct task_info *info = calloc(1, sizeof(struct task_info));
+
+	if (!info)
+		return NULL;
+
+	info->private_data = thread_private;
+	info->threadfn = threadfn;
+	info->postfn = postfn;
+
+	return info;
+}
+
+int task_start(struct task_info *info)
+{
+	int ret;
+
+	if (!info)
+		return -1;
+
+	if (!info->threadfn)
+		return -1;
+
+	ret = pthread_create(&info->id, NULL, info->threadfn,
+			     info->private_data);
+
+	if (ret == 0)
+		pthread_detach(info->id);
+	else
+		info->id = -1;
+
+	return ret;
+}
+
+void task_stop(struct task_info *info)
+{
+	if (!info)
+		return;
+
+	if (info->periodic.timer_fd)
+		close(info->periodic.timer_fd);
+
+	if (info->id > 0)
+		pthread_cancel(info->id);
+
+	if (info->postfn)
+		info->postfn(info->private_data);
+}
+
+void task_deinit(struct task_info *info)
+{
+	if (!info)
+		return;
+
+	free(info);
+}
+
+int task_period_start(struct task_info *info, unsigned int period_ms)
+{
+	unsigned int ns;
+	unsigned int sec;
+	struct itimerspec itval;
+
+	if (!info)
+		return -1;
+
+	info->periodic.timer_fd = timerfd_create(CLOCK_MONOTONIC, 0);
+	if (info->periodic.timer_fd == -1)
+		return info->periodic.timer_fd;
+
+	info->periodic.wakeups_missed = 0;
+
+	sec = period_ms/1000;
+	ns = (period_ms - (sec * 1000)) * 1000;
+	itval.it_interval.tv_sec = sec;
+	itval.it_interval.tv_nsec = ns;
+	itval.it_value.tv_sec = sec;
+	itval.it_value.tv_nsec = ns;
+
+	return timerfd_settime(info->periodic.timer_fd, 0, &itval, NULL);
+};
+
+void task_period_wait(struct task_info *info)
+{
+	unsigned long long missed;
+	int ret;
+
+	if (!info)
+		return;
+
+	ret = read(info->periodic.timer_fd, &missed, sizeof (missed));
+	if (ret == -1) {
+		perror ("read timer");
+		return;
+	}
+
+	if (missed > 0)
+		info->periodic.wakeups_missed += (missed - 1);
+}
+
+void task_period_stop(struct task_info *info)
+{
+	if (!info)
+		return;
+
+	if (info->periodic.timer_fd) {
+		timerfd_settime(info->periodic.timer_fd, 0, NULL, NULL);
+		close(info->periodic.timer_fd);
+	}
+}
task-util.h new file +33 -0
new file mode 100644
index 0000000..95f7b5b
@@ -0,0 +1,33 @@
+
+#ifndef __PROGRESS_
+#define __PROGRESS_
+
+#include <pthread.h>
+
+struct periodic_info {
+	int timer_fd;
+	unsigned long long wakeups_missed;
+};
+
+struct task_info {
+	struct periodic_info periodic;
+	pthread_t id;
+	void *private_data;
+	void *(*threadfn)(void *);
+	int (*postfn)(void *);
+};
+
+/* task life cycle */
+struct task_info *task_init(void *(*threadfn)(void *), int (*postfn)(void *),
+			    void *thread_private);
+int task_start(struct task_info *info);
+void task_stop(struct task_info *info);
+void task_deinit(struct task_info *info);
+
+/* periodic life cycle */
+int task_period_start(struct task_info *info, unsigned int period_ms);
+void task_period_wait(struct task_info *info);
+void task_period_stop(struct task_info *info);
+
+#endif /* __PROGRESS_ */
+
Signature · 1 line
2.1.2
#2 lore raw

Re: [PATCH 1/2] btrfs-progs: add task-utils

Hi,

so started reviewing the patches for inclusion in the 3.18 branch and found a few things that I've fixed locally, this is just FYI.

task-util.c

Quoted 13 lines · Silvio Fricke
---
 task-util.c | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 task-util.h |  33 +++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100644 task-util.c
 create mode 100644 task-util.h
task-util.c new file +0 -0
new file mode 100644
index 0000000..9268df7

Renamed the file to task-utils.c

task-util.h line 3

Quoted 9 lines
task-util.h new file +3 -0
new file mode 100644
index 0000000..95f7b5b
@@ -0,0 +1,33 @@
+
+#ifndef __PROGRESS_
+#define __PROGRESS_

__TASK_UTILS_H__

Quoted 19 lines
task-util.h new file +19 -0
+
+#include <pthread.h>
+
+struct periodic_info {
+	int timer_fd;
+	unsigned long long wakeups_missed;
+};
+
+struct task_info {
+	struct periodic_info periodic;
+	pthread_t id;
+	void *private_data;
+	void *(*threadfn)(void *);
+	int (*postfn)(void *);
+};
+
+/* task life cycle */
+struct task_info *task_init(void *(*threadfn)(void *), int (*postfn)(void *),
+			    void *thread_private);

task-util.h line 32

Quoted 10 lines
task-util.h new file +10 -0
+int task_start(struct task_info *info);
+void task_stop(struct task_info *info);
+void task_deinit(struct task_info *info);
+
+/* periodic life cycle */
+int task_period_start(struct task_info *info, unsigned int period_ms);
+void task_period_wait(struct task_info *info);
+void task_period_stop(struct task_info *info);
+
+#endif /* __PROGRESS_ */

dtto

#3 lore raw

PATCH 2/2 btrfs-progs: convert: use task for progress indication of metadata creation

In reply to Silvio Fricke
---
 Documentation/btrfs-convert.txt |  2 ++
 Makefile                        |  6 ++--
 btrfs-convert.c                 | 64 +++++++++++++++++++++++++++++++++++++----
 3 files changed, 64 insertions(+), 8 deletions(-)
Documentation/btrfs-convert.txt +2 -0
index 555fb35..9cc326c 100644
@@ -29,6 +29,8 @@ Roll back to ext2fs.
 set filesystem label during conversion.
 -L::
 use label from the converted filesystem.
+-p::
+Show progress of convertation.
 
 EXIT STATUS
 -----------
Makefile +3 -3
index 203597c..f76c6b2 100644
@@ -8,7 +8,7 @@ AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DBTRFS_FLAT_INCLUDES -fno-strict-alias
 CFLAGS = -g -O1 -fno-strict-aliasing -rdynamic
 objects = ctree.o disk-io.o radix-tree.o extent-tree.o print-tree.o \
 	  root-tree.o dir-item.o file-item.o inode-item.o inode-map.o \
-	  extent-cache.o extent_io.o volumes.o utils.o repair.o \
+	  extent-cache.o extent_io.o volumes.o utils.o repair.o task-util.o \
 	  qgroup.o raid6.o free-space-cache.o list_sort.o props.o \
 	  ulist.o qgroup-verify.o backref.o
 cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \
@@ -20,13 +20,13 @@ libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
 		   uuid-tree.o utils-lib.o rbtree-utils.o
 libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
 	       crc32c.h list.h kerncompat.h radix-tree.h extent-cache.h \
-	       extent_io.h ioctl.h ctree.h btrfsck.h version.h
+	       extent_io.h ioctl.h ctree.h btrfsck.h version.h task-util.h
 TESTS = fsck-tests.sh convert-tests.sh
 
 INSTALL = install
 prefix ?= /usr/local
 bindir = $(prefix)/bin
-lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L.
+lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. -pthread
 libdir ?= $(prefix)/lib
 incdir = $(prefix)/include/btrfs
 LIBS = $(lib_LIBS) $(libs_static)
btrfs-convert.c +59 -5

132 lines not on this page · raw message

Signature · 1 line
2.1.2
#4 lore raw

Re: [PATCH 2/2] btrfs-progs: convert: use task for progress indication of metadata creation

Fixed locally, no need to resend the patch. JFYI the changes I made

Quoted 11 lines · Silvio Fricke
---
 Documentation/btrfs-convert.txt |  2 ++
 Makefile                        |  6 ++--
 btrfs-convert.c                 | 64 +++++++++++++++++++++++++++++++++++++----
 3 files changed, 64 insertions(+), 8 deletions(-)
Documentation/btrfs-convert.txt +0 -0
index 555fb35..9cc326c 100644

Documentation/btrfs-convert.txt Roll back to ext2fs. · line 33

Quoted 5 lines
Documentation/btrfs-convert.txt +2 -0
@@ -29,6 +29,8 @@ Roll back to ext2fs.
 set filesystem label during conversion.
 -L::
 use label from the converted filesystem.
+-p::
+Show progress of convertation.

conversion

Quoted 16 lines
Documentation/btrfs-convert.txt +0 -0
 EXIT STATUS
 -----------
Makefile +1 -1
index 203597c..f76c6b2 100644
@@ -8,7 +8,7 @@ AM_CFLAGS = -Wall -D_FILE_OFFSET_BITS=64 -DBTRFS_FLAT_INCLUDES -fno-strict-alias
 CFLAGS = -g -O1 -fno-strict-aliasing -rdynamic
 objects = ctree.o disk-io.o radix-tree.o extent-tree.o print-tree.o \
 	  root-tree.o dir-item.o file-item.o inode-item.o inode-map.o \
-	  extent-cache.o extent_io.o volumes.o utils.o repair.o \
+	  extent-cache.o extent_io.o volumes.o utils.o repair.o task-util.o \
 	  qgroup.o raid6.o free-space-cache.o list_sort.o props.o \
 	  ulist.o qgroup-verify.o backref.o
 cmds_objects = cmds-subvolume.o cmds-filesystem.o cmds-device.o cmds-scrub.o \

Makefile libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ · line 23

Quoted 5 lines
Makefile +1 -1
@@ -20,13 +20,13 @@ libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \
 		   uuid-tree.o utils-lib.o rbtree-utils.o
 libbtrfs_headers = send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
 	       crc32c.h list.h kerncompat.h radix-tree.h extent-cache.h \
-	       extent_io.h ioctl.h ctree.h btrfsck.h version.h
+	       extent_io.h ioctl.h ctree.h btrfsck.h version.h task-util.h

It's not used in library, removed

Makefile libbtrfs_objects = send-stream.o send-utils.o rbtree.o btrfs-list.o crc32c.o \ · line 29

Quoted 7 lines
Makefile +1 -1
 TESTS = fsck-tests.sh convert-tests.sh
 
 INSTALL = install
 prefix ?= /usr/local
 bindir = $(prefix)/bin
-lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L.
+lib_LIBS = -luuid -lblkid -lm -lz -llzo2 -L. -pthread

dtto

Quoted 16 lines
Makefile +0 -0
 libdir ?= $(prefix)/lib
 incdir = $(prefix)/include/btrfs
 LIBS = $(lib_LIBS) $(libs_static)
btrfs-convert.c +1 -0
index a544fc6..f8ba920 100644
@@ -38,6 +38,7 @@
 #include "transaction.h"
 #include "crc32c.h"
 #include "utils.h"
+#include "task-util.h"
 #include <ext2fs/ext2_fs.h>
 #include <ext2fs/ext2fs.h>
 #include <ext2fs/ext2_ext_attr.h>

btrfs-convert.c line 49

Quoted 4 lines
btrfs-convert.c +1 -0
@@ -45,6 +46,41 @@
 #define INO_OFFSET (BTRFS_FIRST_FREE_OBJECTID - EXT2_ROOT_INO)
 #define EXT2_IMAGE_SUBVOL_OBJECTID BTRFS_FIRST_FREE_OBJECTID
 
+struct private {

We'd still like to keep the namespace C++ clean, renamed to task_ctx

Quoted 64 lines
btrfs-convert.c +37 -2
+	uint32_t max_copy_inodes;
+	uint32_t cur_copy_inodes;
+	struct task_info *info;
+};
+
+static void *print_copied_inodes(void *p)
+{
+	struct private *priv = p;
+	const char work_indicator[] = { '.', 'o', 'O', 'o' };
+	uint32_t count = 0;
+
+	task_period_start(priv->info, 1000 /* 1s */);
+	while (1) {
+		count++;
+		printf("copy inodes [%c] [%10d/%10d]\r",
+		       work_indicator[count % 4], priv->cur_copy_inodes,
+		       priv->max_copy_inodes);
+		fflush(stdout);
+		task_period_wait(priv->info);
+	}
+
+	return NULL;
+}
+
+static int after_copied_inodes(void *p)
+{
+	struct private *priv = p;
+
+	printf("\n");
+	task_period_stop(priv->info);
+
+	return 0;
+}
+
 /*
  * Open Ext2fs in readonly mode, read block allocation bitmap and
  * inode bitmap into memory.
@@ -1036,7 +1072,7 @@ fail:
  * scan ext2's inode bitmap and copy all used inodes.
  */
 static int copy_inodes(struct btrfs_root *root, ext2_filsys ext2_fs,
-		       int datacsum, int packing, int noxattr)
+		       int datacsum, int packing, int noxattr, struct private *p)
 {
 	int ret;
 	errcode_t err;
@@ -1068,6 +1104,7 @@ static int copy_inodes(struct btrfs_root *root, ext2_filsys ext2_fs,
 					objectid, ext2_fs, ext2_ino,
 					&ext2_inode, datacsum, packing,
 					noxattr);
+		p->cur_copy_inodes++;
 		if (ret)
 			return ret;
 		if (trans->blocks_used >= 4096) {
@@ -2197,7 +2234,7 @@ err:
 }
 
 static int do_convert(const char *devname, int datacsum, int packing, int noxattr,
-	       int copylabel, const char *fslabel)
+	       int copylabel, const char *fslabel, int progress)
 {
 	int i, ret;
 	int fd = -1;

btrfs-convert.c static int do_convert(const char *devname, int datacsum, int packing, int noxatt · line 2315

Quoted 5 lines
btrfs-convert.c +1 -1
@@ -2275,11 +2312,23 @@ static int do_convert(const char *devname, int datacsum, int packing, int noxatt
 		goto fail;
 	}
 	printf("creating btrfs metadata.\n");
-	ret = copy_inodes(root, ext2_fs, datacsum, packing, noxattr);
+	struct private p = {

Declared at the beginning of the function, renamed from a single letter

Quoted 67 lines
btrfs-convert.c +19 -2
+		.max_copy_inodes = (ext2_fs->super->s_inodes_count - ext2_fs->super->s_free_inodes_count),
+		.cur_copy_inodes = 0,
+	};
+	if (progress) {
+		p.info = task_init(print_copied_inodes, after_copied_inodes, &p);
+		task_start(p.info);
+	}
+	ret = copy_inodes(root, ext2_fs, datacsum, packing, noxattr, &p);
 	if (ret) {
 		fprintf(stderr, "error during copy_inodes %d\n", ret);
 		goto fail;
 	}
+	if (progress) {
+		task_stop(p.info);
+		task_deinit(p.info);
+	}
 	printf("creating ext2fs image file.\n");
 	ext2_root = link_subvol(root, "ext2_saved", EXT2_IMAGE_SUBVOL_OBJECTID);
 	if (!ext2_root) {
@@ -2703,6 +2752,7 @@ static void print_usage(void)
 	printf("\t-r           roll back to ext2fs\n");
 	printf("\t-l LABEL     set filesystem label\n");
 	printf("\t-L           use label from converted fs\n");
+	printf("\t-p           show converting progress\n");
 }
 
 int main(int argc, char *argv[])
@@ -2714,11 +2764,12 @@ int main(int argc, char *argv[])
 	int rollback = 0;
 	int copylabel = 0;
 	int usage_error = 0;
+	int progress = 0;
 	char *file;
 	char *fslabel = NULL;
 
 	while(1) {
-		int c = getopt(argc, argv, "dinrl:L");
+		int c = getopt(argc, argv, "dinrl:Lp");
 		if (c < 0)
 			break;
 		switch(c) {
@@ -2747,6 +2798,9 @@ int main(int argc, char *argv[])
 			case 'L':
 				copylabel = 1;
 				break;
+			case 'p':
+				progress = 1;
+				break;
 			default:
 				print_usage();
 				return 1;
@@ -2784,7 +2838,7 @@ int main(int argc, char *argv[])
 	if (rollback) {
 		ret = do_rollback(file);
 	} else {
-		ret = do_convert(file, datacsum, packing, noxattr, copylabel, fslabel);
+		ret = do_convert(file, datacsum, packing, noxattr, copylabel, fslabel, progress);
 	}
 	if (ret)
 		return 1;
Signature · 6 lines
2.1.2

--
To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
#5 lore raw
In reply to Silvio Fricke
Quoted 6 lines · Silvio Fricke

I have tried a ext4->btrfs conversation of a 1TB (90%full) and have seen that btrfs has no progressbar or something like that. Furthermore I have found [1] a project idea for this.

I have tried a generic tasklet-based solution to print the status of the convert-process. Maybe it looks good to you than please add this to your repo.

Added to integration for now to give it some more testing. There are potential users for the 'progress' feature (scrub, balance, replace) so we may build it on top of your patches though I haven't looked closer.