Skip to contentSkip to conversation

Btrfs-progs: send, bump stream version

RFC PATCH Superseded Howard McLauchlan 1 participant 6 messages May 9, 2018 Btrfs file system

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Howard McLauchlan

author
#0 lore raw

RFC PATCH 1/6 Btrfs-progs: send, bump stream version

From: Filipe Manana <[email protected]>

This increases the send stream version from version 1 to version 2, adding new commands:

1) total data size - used to tell the receiver how much file data the stream will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

This doesn't break compatibility with older kernels or clients. In order to get a version 2 send stream, new flags must be passed to the send ioctl.

Signed-off-by Filipe David Borba Manana <[email protected]>
---
 cmds-send.c   | 61 ++++++++++++++++++++++++++++++++++++---------------
 ioctl.h       | 15 +++++++++++++
 send-stream.c |  2 +-
 send.h        | 23 ++++++++++++++++++-
 4 files changed, 81 insertions(+), 20 deletions(-)
cmds-send.c +43 -18
index c5ecdaa1..0ec557c7 100644
@@ -52,6 +52,7 @@
  * the 'At subvol' message.
  */
 static int g_verbose = 1;
+static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
 
 struct btrfs_send {
 	int send_fd;

@@ -343,6 +344,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_STREAM_HEADER;
 	if (!is_last_subvol)
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
+	if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
+		io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret < 0) {
 		ret = -errno;

@@ -513,7 +516,8 @@ int cmd_send(int argc, char **argv)
 		static const struct option long_options[] = {
 			{ "verbose", no_argument, NULL, 'v' },
 			{ "quiet", no_argument, NULL, 'q' },
-			{ "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA }
+			{ "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA },
+			{ "stream-version", 1, NULL, 'V' },
 		};
 		int c = getopt_long(argc, argv, "vqec:f:i:p:", long_options, NULL);
 

@@ -597,6 +601,24 @@ int cmd_send(int argc, char **argv)
 			error("option -i was removed, use -c instead");
 			ret = 1;
 			goto out;
+		case 'V':
+			if (sscanf(optarg, "%d", &g_stream_version) != 1) {
+				fprintf(stderr,
+					"ERROR: invalid value for stream version: %s\n",
+					optarg);
+				ret = 1;
+				goto out;
+			}
+			if (g_stream_version <= 0 ||
+			    g_stream_version > BTRFS_SEND_STREAM_VERSION_MAX) {
+				fprintf(stderr,
+					"ERROR: unsupported stream version %d, minimum: 1, maximum: %d\n",
+					g_stream_version,
+					BTRFS_SEND_STREAM_VERSION_MAX);
+				ret = 1;
+				goto out;
+			}
+			break;
 		case GETOPT_VAL_SEND_NO_DATA:
 			send_flags |= BTRFS_SEND_FLAG_NO_FILE_DATA;
 			break;

@@ -776,7 +798,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-ve] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"<subvol> should be read-only here.",

@@ -790,21 +812,24 @@ const char * const cmd_send_usage[] = {
 	"which case 'btrfs send' will determine a suitable parent among the",
 	"clone sources itself.",
 	"\n",
-	"-e               If sending multiple subvols at once, use the new",
-	"                 format and omit the end-cmd between the subvols.",
-	"-p <parent>      Send an incremental stream from <parent> to",
-	"                 <subvol>.",
-	"-c <clone-src>   Use this snapshot as a clone source for an ",
-	"                 incremental send (multiple allowed)",
-	"-f <outfile>     Output is normally written to stdout. To write to",
-	"                 a file, use this option. An alternative would be to",
-	"                 use pipes.",
-	"--no-data        send in NO_FILE_DATA mode, Note: the output stream",
-	"                 does not contain any file data and thus cannot be used",
-	"                 to transfer changes. This mode is faster and useful to",
-	"                 show the differences in metadata.",
-	"-v|--verbose     enable verbose output to stderr, each occurrence of",
-	"                 this option increases verbosity",
-	"-q|--quiet       suppress all messages, except errors",
+	"-e                          If sending multiple subvols at once, use the new",
+	"                            format and omit the end-cmd between the subvols.",
+	"-p <parent>                 Send an incremental stream from <parent> to",
+	"                            <subvol>.",
+	"-c <clone-src>              Use this snapshot as a clone source for an ",
+	"                            incremental send (multiple allowed)",
+	"-f <outfile>                Output is normally written to stdout. To write to",
+	"                            a file, use this option. An alternative would be to",
+	"                            use pipes.",
+	"--no-data                   send in NO_FILE_DATA mode, Note: the output stream",
+	"                            does not contain any file data and thus cannot be used",
+	"                            to transfer changes. This mode is faster and useful to",
+	"                            show the differences in metadata.",
+	"-v|--verbose                enable verbose output to stderr, each occurrence of",
+	"                            this option increases verbosity",
+	"-q|--quiet       	     suppress all messages, except errors",
+	"--stream-version <version>  Ask the kernel to produce a specific send stream",
+	"                            version. More recent stream versions provide new",
+	"                            features and better performance. Default value is 1.",
 	NULL
 };
ioctl.h +15 -0
index 709e996f..9d490aca 100644
@@ -632,6 +632,21 @@ BUILD_ASSERT(sizeof(struct btrfs_ioctl_received_subvol_args_32) == 192);
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
 	 BTRFS_SEND_FLAG_OMIT_END_CMD)
 
+/*
+ * The sum of all length fields the receiver will get in write, clone and
+ * fallocate commands.
+ * This can be used by the receiver to compute progress, at the expense of some
+ * initial metadata scan performed by the sender (kernel).
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2               0x10
+
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */
 	__u64 clone_sources_count;	/* in */
send-stream.c +1 -1
index 78f2571a..86956d28 100644
@@ -503,7 +503,7 @@ int btrfs_read_and_process_send_stream(int fd,
 	}
 
 	sctx.version = le32_to_cpu(hdr.version);
-	if (sctx.version > BTRFS_SEND_STREAM_VERSION) {
+	if (sctx.version > BTRFS_SEND_STREAM_VERSION_MAX) {
 		ret = -EINVAL;
 		error("stream version %d not supported, please use newer version",
 				sctx.version);
send.h +22 -1
index fe613cbb..eb14fba3 100644
@@ -27,7 +27,10 @@ extern "C" {
 #endif
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
+/* Max supported stream version. */
+#define BTRFS_SEND_STREAM_VERSION_MAX BTRFS_SEND_STREAM_VERSION_2
 
 #define BTRFS_SEND_BUF_SIZE SZ_64K
 #define BTRFS_SEND_READ_SIZE (1024 * 48)

@@ -94,6 +97,15 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/*
+	 * The following commands were added in stream version 2.
+	 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+	BTRFS_SEND_C_INODE_SET_FLAGS,
+	BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)

@@ -132,10 +144,19 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/*
+	 * The following attributes were added in stream version 2.
+	 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
+	BTRFS_SEND_A_INODE_FLAGS,     /* 32 bits */
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif
Signature · 1 line
2.17.0
author
#1 lore raw

RFC PATCH 2/6 Btrfs-progs: send, implement total data size callback and progress report

From: Filipe Manana <[email protected]>

This is a followup to the kernel patch titled:

Btrfs: send, implement total data size command to allow for progress estimation

This makes the btrfs send and receive commands aware of the new send flag, named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data that is new between the parent and send snapshots/roots. As this command immediately follows the commands to start a snapshot/subvolume, it can be used to report and compute progress, by keeping a counter that is incremented with the data length of each write, clone and fallocate command that is received from the stream.

Example:

    $ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s
    $ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

At the moment progress is only reported by btrfs-receive, but it is possible and simple to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send output to btrfs-receive (directly to a file).

Signed-off-by Filipe David Borba Manana <[email protected]>
---
 cmds-receive.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++
 cmds-send.c    | 23 +++++++++++--
 send-stream.c  |  4 +++
 send-stream.h  |  1 +
 4 files changed, 117 insertions(+), 2 deletions(-)
cmds-receive.c +91 -0
index 68123a31..d8ff5194 100644
@@ -30,6 +30,7 @@
 #include <assert.h>
 #include <getopt.h>
 #include <limits.h>
+#include <time.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>

@@ -79,6 +80,14 @@ struct btrfs_receive
 
 	int honor_end_cmd;
 
+	/* For the subvolume/snapshot we're currently receiving. */
+	u64 total_data_size;
+	u64 bytes_received;
+	time_t last_progress_update;
+	u64 bytes_received_last_update;
+	float progress;
+	const char *target;
+
 	/*
 	 * Buffer to store capabilities from security.capabilities xattr,
 	 * usually 20 bytes, but make same room for potentially larger

@@ -156,6 +165,16 @@ out:
 	return ret;
 }
 
+static void reset_progress(struct btrfs_receive *rctx, const char *dest)
+{
+	rctx->total_data_size = 0;
+	rctx->bytes_received = 0;
+	rctx->progress = 0.0;
+	rctx->last_progress_update = 0;
+	rctx->bytes_received_last_update = 0;
+	rctx->target = dest;
+}
+
 static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 			  void *user)
 {

@@ -180,6 +199,7 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid,
 		ret = -EINVAL;
 		goto out;
 	}
+	reset_progress(rctx, "Subvolume");
 
 	if (*rctx->dest_dir_path == 0) {
 		strncpy_null(rctx->cur_subvol_path, path);

@@ -249,6 +269,7 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid,
 		ret = -EINVAL;
 		goto out;
 	}
+	reset_progress(rctx, "Snapshot");
 
 	if (*rctx->dest_dir_path == 0) {
 		strncpy_null(rctx->cur_subvol_path, path);

@@ -388,6 +409,73 @@ out:
 	return ret;
 }
 
+static int process_total_data_size(u64 size, void *user)
+{
+	struct btrfs_receive *rctx = user;
+
+	rctx->total_data_size = size;
+	fprintf(stdout, "About to receive %llu bytes\n", size);
+
+	return 0;
+}
+
+static void update_progress(struct btrfs_receive *rctx, u64 bytes)
+{
+	float new_progress;
+	time_t now;
+	time_t tdiff;
+
+	if (rctx->total_data_size == 0)
+		return;
+
+	rctx->bytes_received += bytes;
+
+	now = time(NULL);
+	tdiff = now - rctx->last_progress_update;
+	if (tdiff < 1) {
+		if (rctx->bytes_received == rctx->total_data_size)
+			fprintf(stdout, "\n");
+		return;
+	}
+
+	new_progress = ((float)rctx->bytes_received / rctx->total_data_size) * 100.0;
+
+	if ((int)(new_progress * 100) > (int)(rctx->progress * 100) ||
+	    rctx->bytes_received == rctx->total_data_size) {
+		char line[5000];
+		float rate = rctx->bytes_received - rctx->bytes_received_last_update;
+		const char *rate_units;
+
+		rate /= tdiff;
+		if (rate > (1024 * 1024)) {
+			rate_units = "MB/s";
+			rate /= 1024 * 1024;
+		} else if (rate > 1024) {
+			rate_units = "KB/s";
+			rate /= 1024;
+		} else {
+			rate_units = "B/s";
+		}
+
+		snprintf(line, sizeof(line),
+			 "%s%s %s, %llu / %llu bytes received, %5.2f%%, %5.2f%s%s",
+			 (g_verbose ? "" : "\r"),
+			 rctx->target,
+			 rctx->full_subvol_path,
+			 rctx->bytes_received, rctx->total_data_size,
+			 new_progress, rate, rate_units,
+			 (g_verbose ? "\n" : ""));
+		fprintf(stdout, "%s%s", line, (g_verbose ? "" : "        "));
+		fflush(stdout);
+	}
+
+	if (rctx->bytes_received == rctx->total_data_size)
+		fprintf(stdout, "\n");
+	rctx->progress = new_progress;
+	rctx->last_progress_update = now;
+	rctx->bytes_received_last_update = rctx->bytes_received;
+}
+
 static int process_mkfile(const char *path, void *user)
 {
 	int ret;

@@ -722,6 +810,7 @@ static int process_write(const char *path, const void *data, u64 offset,
 		}
 		pos += w;
 	}
+	update_progress(rctx, len);
 
 out:
 	return ret;

@@ -827,6 +916,7 @@ static int process_clone(const char *path, u64 offset, u64 len,
 				path, strerror(-ret));
 		goto out;
 	}
+	update_progress(rctx, len);
 
 out:
 	if (si) {

@@ -1081,6 +1171,7 @@ static struct btrfs_send_ops send_ops = {
 	.chown = process_chown,
 	.utimes = process_utimes,
 	.update_extent = process_update_extent,
+	.total_data_size = process_total_data_size,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
cmds-send.c +21 -2
index 0ec557c7..45e30f53 100644
@@ -53,6 +53,7 @@
  */
 static int g_verbose = 1;
 static int g_stream_version = BTRFS_SEND_STREAM_VERSION_1;
+static int g_total_data_size = 0;
 
 struct btrfs_send {
 	int send_fd;

@@ -346,6 +347,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id,
 		io_send.flags |= BTRFS_SEND_FLAG_OMIT_END_CMD;
 	if (g_stream_version == BTRFS_SEND_STREAM_VERSION_2)
 		io_send.flags |= BTRFS_SEND_FLAG_STREAM_V2;
+	if (g_total_data_size)
+		io_send.flags |= BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE;
 	ret = ioctl(subvol_fd, BTRFS_IOC_SEND, &io_send);
 	if (ret < 0) {
 		ret = -errno;

@@ -519,7 +522,7 @@ int cmd_send(int argc, char **argv)
 			{ "no-data", no_argument, NULL, GETOPT_VAL_SEND_NO_DATA },
 			{ "stream-version", 1, NULL, 'V' },
 		};
-		int c = getopt_long(argc, argv, "vqec:f:i:p:", long_options, NULL);
+		int c = getopt_long(argc, argv, "vqesc:f:i:p:", long_options, NULL);
 
 		if (c < 0)
 			break;

@@ -619,6 +622,9 @@ int cmd_send(int argc, char **argv)
 				goto out;
 			}
 			break;
+		case 's':
+			g_total_data_size = 1;
+			break;
 		case GETOPT_VAL_SEND_NO_DATA:
 			send_flags |= BTRFS_SEND_FLAG_NO_FILE_DATA;
 			break;

@@ -633,6 +639,14 @@ int cmd_send(int argc, char **argv)
 	if (check_argc_min(argc - optind, 1))
 		usage(cmd_send_usage);
 
+	if (g_total_data_size &&
+	    g_stream_version < BTRFS_SEND_STREAM_VERSION_2) {
+		fprintf(stderr,
+			"ERROR: option total data size (-s) requires use of the send stream version 2 or higher\n");
+		ret = 1;
+		goto out;
+	}
+
 	if (outname[0]) {
 		int tmpfd;
 

@@ -798,7 +812,7 @@ out:
 }
 
 const char * const cmd_send_usage[] = {
-	"btrfs send [-ve] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
+	"btrfs send [-ves] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]",
 	"Send the subvolume(s) to stdout.",
 	"Sends the subvolume(s) specified by <subvol> to stdout.",
 	"<subvol> should be read-only here.",

@@ -831,5 +845,10 @@ const char * const cmd_send_usage[] = {
 	"--stream-version <version>  Ask the kernel to produce a specific send stream",
 	"                            version. More recent stream versions provide new",
 	"                            features and better performance. Default value is 1.",
+	"-s                          Obtain the total data size for each subvolume or ",
+	"                            snapshot to send. This demands additional processing",
+	"                            (mostly IO bound) but is useful for the receive ",
+	"                            command to report progress. This option requires use",
+	"                            of the send stream version 2 or higher.",
 	NULL
 };
send-stream.c +4 -0
index 86956d28..d30fd5a7 100644
@@ -453,6 +453,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx)
 		TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, &tmp);
 		ret = sctx->ops->update_extent(path, offset, tmp, sctx->user);
 		break;
+	case BTRFS_SEND_C_TOTAL_DATA_SIZE:
+		TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, &tmp);
+		ret = sctx->ops->total_data_size(tmp, sctx->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
send-stream.h +1 -0
index 39901f86..5b244ab6 100644
@@ -66,6 +66,7 @@ struct btrfs_send_ops {
 		      struct timespec *mt, struct timespec *ct,
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
+	int (*total_data_size)(u64 size, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
2.17.0
author
#2 lore raw

RFC PATCH 3/6 Btrfs-progs: send, implement fallocate command callback

In reply to Howard McLauchlan

From: Filipe Manana <[email protected]>

The fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).

Signed-off-by Filipe David Borba Manana <[email protected]>
---
 cmds-receive.c | 44 ++++++++++++++++++++++++++++++++++++++++++++
 send-stream.c  | 13 +++++++++++++
 send-stream.h  |  2 ++
 3 files changed, 59 insertions(+)
cmds-receive.c +44 -0
index d8ff5194..510b6bc8 100644
@@ -39,6 +39,7 @@
 #include <sys/types.h>
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
+#include <linux/falloc.h>
 
 #include "ctree.h"
 #include "ioctl.h"

@@ -1149,6 +1150,48 @@ static int process_update_extent(const char *path, u64 offset, u64 len,
 	return 0;
 }
 
+static int process_fallocate(const char *path, u32 flags, u64 offset,
+			     u64 len, void *user)
+{
+	struct btrfs_receive *rctx = user;
+	int mode = 0;
+	int ret;
+	char full_path[PATH_MAX];
+
+	ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+	if (ret < 0) {
+		error("fallocate: path invalid: %s", path);
+		goto out;
+	}
+
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE)
+		mode |= FALLOC_FL_KEEP_SIZE;
+	if (flags & BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+		mode |= FALLOC_FL_PUNCH_HOLE;
+
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"fallocate %s - flags %u, offset %llu, len %llu\n",
+			path, flags, offset, len);
+
+	ret = open_inode_for_write(rctx, full_path);
+	if (ret < 0)
+		goto out;
+
+	ret = fallocate(rctx->write_fd, mode, offset, len);
+	if (ret) {
+		ret = -errno;
+		fprintf(stderr,
+			"ERROR: fallocate against %s failed. %s\n",
+			path, strerror(-ret));
+		goto out;
+	}
+	update_progress(rctx, len);
+
+out:
+	return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
 	.snapshot = process_snapshot,

@@ -1172,6 +1215,7 @@ static struct btrfs_send_ops send_ops = {
 	.utimes = process_utimes,
 	.update_extent = process_update_extent,
 	.total_data_size = process_total_data_size,
+	.fallocate = process_fallocate,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
send-stream.c +13 -0
index d30fd5a7..74ec37dd 100644
@@ -457,6 +457,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx)
 		TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, &tmp);
 		ret = sctx->ops->total_data_size(tmp, sctx->user);
 		break;
+	case BTRFS_SEND_C_FALLOCATE:
+		{
+			u32 flags;
+			u64 len;
+
+			TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path);
+			TLV_GET_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, &flags);
+			TLV_GET_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, &offset);
+			TLV_GET_U64(sctx, BTRFS_SEND_A_SIZE, &len);
+			ret = sctx->ops->fallocate(path, flags, offset, len,
+						sctx->user);
+		}
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
send-stream.h +2 -0
index 5b244ab6..89e64043 100644
@@ -67,6 +67,8 @@ struct btrfs_send_ops {
 		      void *user);
 	int (*update_extent)(const char *path, u64 offset, u64 len, void *user);
 	int (*total_data_size)(u64 size, void *user);
+	int (*fallocate)(const char *path, u32 flags, u64 offset,
+			 u64 len, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
2.17.0
author
#3 lore raw

RFC PATCH 4/6 Btrfs-progs: add write and clone commands debug info to receive

In reply to Howard McLauchlan

From: Filipe Manana <[email protected]>

When specifying -vv print information about received write and clone commands too, as we do this for other commands already and it's very useful for debugging and troubleshooting.

Signed-off-by Filipe David Borba Manana <[email protected]>
---
 cmds-receive.c | 9 +++++++++
 1 file changed, 9 insertions(+)
cmds-receive.c +9 -0
index 510b6bc8..20e593f7 100644
@@ -790,6 +790,10 @@ static int process_write(const char *path, const void *data, u64 offset,
 	u64 pos = 0;
 	int w;
 
+	if (g_verbose >= 2)
+		fprintf(stderr, "write %s, offset %llu, len %llu\n",
+			path, offset, len);
+
 	ret = path_cat_out(full_path, rctx->full_subvol_path, path);
 	if (ret < 0) {
 		error("write: path invalid: %s", path);

@@ -831,6 +835,11 @@ static int process_clone(const char *path, u64 offset, u64 len,
 	char full_clone_path[PATH_MAX];
 	int clone_fd = -1;
 
+	if (g_verbose >= 2)
+		fprintf(stderr,
+			"clone %s, offset %llu, len %llu, clone path %s, clone offset %llu\n",
+			path, offset, len, clone_path, clone_offset);
+
 	ret = path_cat_out(full_path, rctx->full_subvol_path, path);
 	if (ret < 0) {
 		error("clone: source path invalid: %s", path);
Signature · 1 line
2.17.0
author
#4 lore raw

RFC PATCH 5/6 btrfs-progs: add total data size, fallocate to dump

In reply to Howard McLauchlan

From: Howard McLauchlan <[email protected]>

Adding entries to dump for new commands (total data size, fallocate).

Signed-off-by Howard McLauchlan <[email protected]>
---
 send-dump.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)
send-dump.c +18 -1
index 1591e0cc..c5a695a2 100644
@@ -316,6 +316,21 @@ static int print_update_extent(const char *path, u64 offset, u64 len,
 			  offset, len);
 }
 
+static int print_total_data_size(u64 size, void *user)
+{
+	char path;
+
+	return PRINT_DUMP(user, &path, "total_data_size", "size=%llu", size);
+}
+
+static int print_fallocate(const char *path, u32 flags, u64 offset, u64 len,
+			   void *user)
+{
+	return PRINT_DUMP(user, path, "fallocate",
+			  "flags=%u offset=%llu len=%llu", flags, offset,
+			  len);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
 	.subvol = print_subvol,
 	.snapshot = print_snapshot,

@@ -337,5 +352,7 @@ struct btrfs_send_ops btrfs_print_send_ops = {
 	.chmod = print_chmod,
 	.chown = print_chown,
 	.utimes = print_utimes,
-	.update_extent = print_update_extent
+	.update_extent = print_update_extent,
+	.total_data_size = print_total_data_size,
+	.fallocate = print_fallocate,
 };
Signature · 1 line
2.17.0
author
#5 lore raw

RFC PATCH 6/6 btrfs-progs: add chattr support for send/receive

In reply to Howard McLauchlan

From: Howard McLauchlan <[email protected]>

Presently, btrfs send/receive does not propagate inode attribute flags; all chattr operations are effectively discarded upon transmission.

This patch adds userspace support for inode attribute flags. Kernel support can be found under the commit:

btrfs: add chattr support for send/receive

An associated xfstest can also be found at:

btrfs: verify chattr support for send/receive test

Signed-off-by Howard McLauchlan <[email protected]>
---
 cmds-receive.c | 37 +++++++++++++++++++++++++++++++++++++
 send-dump.c    |  6 ++++++
 send-stream.c  |  5 +++++
 send-stream.h  |  1 +
 send.h         |  4 ++--
 5 files changed, 51 insertions(+), 2 deletions(-)
cmds-receive.c +37 -0
index 20e593f7..2a841bfc 100644
@@ -40,6 +40,7 @@
 #include <sys/xattr.h>
 #include <uuid/uuid.h>
 #include <linux/falloc.h>
+#include <linux/fs.h>
 
 #include "ctree.h"
 #include "ioctl.h"

@@ -1201,6 +1202,41 @@ out:
 	return ret;
 }
 
+static int process_chattr(const char *path, u64 flags, void *user)
+{
+	int ret = 0;
+	int fd = 0;
+	int _flags = flags;
+	struct btrfs_receive *rctx = user;
+	char full_path[PATH_MAX];
+
+	ret = path_cat_out(full_path, rctx->full_subvol_path, path);
+	if (ret < 0) {
+		error("chattr: path invalid: %s", path);
+		goto out;
+	}
+
+	if (g_verbose >= 2)
+		fprintf(stderr, "chattr %s - flags=0%o\n", path, (int)flags);
+
+	fd = open(full_path, O_RDONLY);
+	if (fd < 0) {
+		ret = -errno;
+		error("cannot open %s: %s", path, strerror(-ret));
+		goto out;
+	}
+
+	ret = ioctl(fd, FS_IOC_SETFLAGS, &_flags);
+	if (ret < 0) {
+		ret = -errno;
+		error("chattr %s failed: %s", path, strerror(-ret));
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
 static struct btrfs_send_ops send_ops = {
 	.subvol = process_subvol,
 	.snapshot = process_snapshot,

@@ -1225,6 +1261,7 @@ static struct btrfs_send_ops send_ops = {
 	.update_extent = process_update_extent,
 	.total_data_size = process_total_data_size,
 	.fallocate = process_fallocate,
+	.chattr = process_chattr,
 };
 
 static int do_receive(struct btrfs_receive *rctx, const char *tomnt,
send-dump.c +6 -0
index c5a695a2..15aea402 100644
@@ -331,6 +331,11 @@ static int print_fallocate(const char *path, u32 flags, u64 offset, u64 len,
 			  len);
 }
 
+static int print_chattr(const char *path, u64 flags, void *user)
+{
+	return PRINT_DUMP(user, path, "chattr", "flags=%llu", flags);
+}
+
 struct btrfs_send_ops btrfs_print_send_ops = {
 	.subvol = print_subvol,
 	.snapshot = print_snapshot,

@@ -355,4 +360,5 @@ struct btrfs_send_ops btrfs_print_send_ops = {
 	.update_extent = print_update_extent,
 	.total_data_size = print_total_data_size,
 	.fallocate = print_fallocate,
+	.chattr = print_chattr,
 };
send-stream.c +5 -0
index 74ec37dd..4f26fae3 100644
@@ -470,6 +470,11 @@ static int read_and_process_cmd(struct btrfs_send_stream *sctx)
 						sctx->user);
 		}
 		break;
+	case BTRFS_SEND_C_CHATTR:
+		TLV_GET_STRING(sctx, BTRFS_SEND_A_PATH, &path);
+		TLV_GET_U64(sctx, BTRFS_SEND_A_CHATTR, &tmp);
+		ret = sctx->ops->chattr(path, tmp, sctx->user);
+		break;
 	case BTRFS_SEND_C_END:
 		ret = 1;
 		break;
send-stream.h +1 -0
index 89e64043..a9f08d52 100644
@@ -69,6 +69,7 @@ struct btrfs_send_ops {
 	int (*total_data_size)(u64 size, void *user);
 	int (*fallocate)(const char *path, u32 flags, u64 offset,
 			 u64 len, void *user);
+	int (*chattr)(const char *path, u64 flags, void *user);
 };
 
 int btrfs_read_and_process_send_stream(int fd,
send.h +2 -2
index eb14fba3..5b5ada61 100644
@@ -103,8 +103,8 @@ enum btrfs_send_cmd {
 	 */
 	BTRFS_SEND_C_TOTAL_DATA_SIZE,
 	BTRFS_SEND_C_FALLOCATE,
-	BTRFS_SEND_C_INODE_SET_FLAGS,
 	BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+	BTRFS_SEND_C_CHATTR,
 
 	__BTRFS_SEND_C_MAX,
 };

@@ -148,7 +148,7 @@ enum {
 	 * The following attributes were added in stream version 2.
 	 */
 	BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
-	BTRFS_SEND_A_INODE_FLAGS,     /* 32 bits */
+	BTRFS_SEND_A_CHATTR,	      /* 32 bits */
 
 	__BTRFS_SEND_A_MAX,
 };
Signature · 1 line
2.17.0