Btrfs: send, bump stream version

PATCH In review Filipe David Borba Manana 4 participants 23 messages Apr 15, 2014 – Jun 23, 2014 Btrfs file system

lore

Share & tools

lore

Filipe David Borba Manana

author

2014-04-15 15:40 UTC #0 lore raw

PATCH 1/4 Btrfs: send, bump stream version

This increases the send stream version from version 1 to version 2, adding 2 new commands:

1) total data size - used to tell the receiver how much file data the stream will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files.

This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).

Signed-off-by Filipe David Borba Manana <[email protected]>

---
 fs/btrfs/send.h            | 13 ++++++++++++-
 include/uapi/linux/btrfs.h | 13 ++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

fs/btrfs/send.h +12 -1

index 48d425a..7b5df08 100644
@@ -20,7 +20,7 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)

@@ -87,6 +87,11 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/* added in stream version 2 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)

@@ -125,10 +130,16 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/* added in stream version 2 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS,
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

include/uapi/linux/btrfs.h +12 -1

index b4d6909..444893d 100644
@@ -464,10 +464,21 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
 #define BTRFS_SEND_FLAG_MASK \
 	(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-	 BTRFS_SEND_FLAG_OMIT_END_CMD)
+	 BTRFS_SEND_FLAG_OMIT_END_CMD | \
+	 BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE)
 
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-15 15:40 UTC #1 lore raw

PATCH 2/4 Btrfs: send, implement total data size command to allow for progress estimation

This new send flag makes send calculate first the amount of new file data (in bytes) the send root has relatively to the parent root, or for the case of a non-incremental send, the total amount of file data the stream will create (including holes and prealloc extents). In other words, it computes the sum of the lengths of all write, clone and fallocate operations that will be sent through the send stream.

This data size value is sent in a new command, named BTRFS_SEND_C_TOTAL_DATA_SIZE, that immediately follows a BTRFS_SEND_C_SUBVOL or BTRFS_SEND_C_SNAPSHOT command, and precedes any command that changes a file or the filesystem hierarchy. Upon receiving a write, clone or fallocate command, the receiving end can increment a counter by the data length of that command and therefore report progress by comparing the counter's value with the data size value received in the BTRFS_SEND_C_TOTAL_DATA_SIZE command.

The approach is simple, before the normal operation of send, do a scan in the file system tree for new inodes and new/changed file extent items, just like in send's normal operation, and keep incrementing a counter with new inodes' size and the size of file extents (and file holes) that are going to be written, cloned or fallocated. This is actually a simpler and more lightweight tree scan/processing than the one we do when sending the changes, as it doesn't process inode references nor does any lookups in the extent tree for example.

After modifying btrfs-progs to understand this new command and report progress, here's an example (the -o flag tells btrfs send to pass the new flag to the kernel's send ioctl):

    $ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

Signed-off-by Filipe David Borba Manana <[email protected]>

---
 fs/btrfs/send.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 162 insertions(+), 32 deletions(-)

fs/btrfs/send.c +162 ~~-32~~

index 289e9f3..0cff503 100644
@@ -81,7 +81,13 @@ struct clone_root {
 #define SEND_CTX_MAX_NAME_CACHE_SIZE 128
 #define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2)
 
+enum btrfs_send_phase {
+	SEND_PHASE_STREAM_CHANGES,
+	SEND_PHASE_COMPUTE_DATA_SIZE,
+};
+
 struct send_ctx {
+	enum btrfs_send_phase phase;
 	struct file *send_filp;
 	loff_t send_off;
 	char *send_buf;

@@ -116,6 +122,7 @@ struct send_ctx {
 	u64 cur_inode_last_extent;
 
 	u64 send_progress;
+	u64 total_data_size;
 
 	struct list_head new_refs;
 	struct list_head deleted_refs;

@@ -688,6 +695,8 @@ static int send_rename(struct send_ctx *sctx,
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_rename %s -> %s\n", from->start, to->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);

@@ -712,6 +721,8 @@ static int send_link(struct send_ctx *sctx,
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_link %s -> %s\n", path->start, lnk->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);

@@ -735,6 +746,8 @@ static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_unlink %s\n", path->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);

@@ -757,6 +770,8 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_rmdir %s\n", path->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);

@@ -2304,6 +2319,9 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
 	int ret = 0;
 	struct fs_path *p;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 verbose_printk("btrfs: send_truncate %llu size=%llu\n", ino, size);
 
 	p = fs_path_alloc();

@@ -2333,6 +2351,8 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_chmod %llu mode=%llu\n", ino, mode);
 
 	p = fs_path_alloc();

@@ -2362,6 +2382,8 @@ static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_chown %llu uid=%llu, gid=%llu\n", ino, uid, gid);
 
 	p = fs_path_alloc();

@@ -2397,6 +2419,8 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
 	struct btrfs_key key;
 	int slot;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_utimes %llu\n", ino);
 
 	p = fs_path_alloc();

@@ -2459,6 +2483,8 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino)
 	u64 mode;
 	u64 rdev;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_create_inode %llu\n", ino);
 
 	p = fs_path_alloc();

@@ -2606,6 +2632,8 @@ static int send_create_inode_if_needed(struct send_ctx *sctx)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	if (S_ISDIR(sctx->cur_inode_mode)) {
 		ret = did_create_dir(sctx, sctx->cur_ino);
 		if (ret < 0)

@@ -2711,6 +2739,8 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
 	int ret;
 	struct fs_path *orphan;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	orphan = fs_path_alloc();
 	if (!orphan)
 		return -ENOMEM;

@@ -3079,6 +3109,8 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
 	int ret;
 	u64 ancestor = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	name = fs_path_alloc();
 	from_path = fs_path_alloc();
 	if (!name || !from_path) {

@@ -3333,6 +3365,9 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
 	int is_orphan = 0;
 	u64 last_dir_ino_rm = 0;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 verbose_printk("btrfs: process_recorded_refs %llu\n", sctx->cur_ino);
 
 	/*

@@ -3841,6 +3876,8 @@ static int process_all_refs(struct send_ctx *sctx,
 	iterate_inode_ref_t cb;
 	int pending_move = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	path = alloc_path_for_send();
 	if (!path)
 		return -ENOMEM;

@@ -4160,6 +4197,8 @@ static int process_all_new_xattrs(struct send_ctx *sctx)
 	struct extent_buffer *eb;
 	int slot;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	path = alloc_path_for_send();
 	if (!path)
 		return -ENOMEM;

@@ -4290,6 +4329,8 @@ static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
 	struct fs_path *p;
 	ssize_t num_read = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4325,6 +4366,22 @@ out:
 	return num_read;
 }
 
+static int send_total_data_size(struct send_ctx *sctx, u64 data_size)
+{
+	int ret;
+
+	ret = begin_cmd(sctx, BTRFS_SEND_C_TOTAL_DATA_SIZE);
+	if (ret < 0)
+		goto out;
+
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, data_size);
+	ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+	return ret;
+}
+
 /*
  * Send a clone command to user space.
  */

@@ -4336,6 +4393,8 @@ static int send_clone(struct send_ctx *sctx,
 	struct fs_path *p;
 	u64 gen;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_clone offset=%llu, len=%d, clone_root=%llu, "
 	       "clone_inode=%llu, clone_offset=%llu\n", offset, len,
 		clone_root->root->objectid, clone_root->ino,

@@ -4394,6 +4453,8 @@ static int send_update_extent(struct send_ctx *sctx,
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4425,6 +4486,11 @@ static int send_hole(struct send_ctx *sctx, u64 end)
 	u64 len;
 	int ret = 0;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		sctx->total_data_size += end - offset;
+		return 0;
+	}
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4488,6 +4554,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		goto out;
 	}
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		if (offset < sctx->cur_inode_size)
+			sctx->total_data_size += len;
+		goto out;
+	}
+
 	if (clone_root && IS_ALIGNED(offset + len, bs)) {
 		ret = send_clone(sctx, offset, len, clone_root);
 	} else if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) {

@@ -4821,10 +4893,12 @@ static int process_extent(struct send_ctx *sctx,
 		}
 	}
 
-	ret = find_extent_clone(sctx, path, key->objectid, key->offset,
-			sctx->cur_inode_size, &found_clone);
-	if (ret != -ENOENT && ret < 0)
-		goto out;
+	if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+		ret = find_extent_clone(sctx, path, key->objectid, key->offset,
+					sctx->cur_inode_size, &found_clone);
+		if (ret != -ENOENT && ret < 0)
+			goto out;
+	}
 
 	ret = send_write_or_clone(sctx, path, key, found_clone);
 	if (ret)

@@ -4954,6 +5028,9 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 	if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
 		goto out;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		goto truncate_inode;
+
 	ret = get_inode_info(sctx->send_root, sctx->cur_ino, NULL, NULL,
 			&left_mode, &left_uid, &left_gid, NULL);
 	if (ret < 0)

@@ -4976,6 +5053,7 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 			need_chmod = 1;
 	}
 
+truncate_inode:
 	if (S_ISREG(sctx->cur_inode_mode)) {
 		if (need_send_hole(sctx)) {
 			if (sctx->cur_inode_last_extent == (u64)-1 ||

@@ -5015,7 +5093,8 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 	 * If other directory inodes depended on our current directory
 	 * inode's move/rename, now do their move/rename operations.
 	 */
-	if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
+	if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE &&
+	    !is_waiting_for_move(sctx, sctx->cur_ino)) {
 		ret = apply_children_dir_moves(sctx);
 		if (ret)
 			goto out;

@@ -5099,7 +5178,8 @@ static int changed_inode(struct send_ctx *sctx,
 				sctx->left_path->nodes[0], left_ii);
 		sctx->cur_inode_rdev = btrfs_inode_rdev(
 				sctx->left_path->nodes[0], left_ii);
-		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
+		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID &&
+		    sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE)
 			ret = send_create_inode_if_needed(sctx);
 	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
 		sctx->cur_inode_gen = right_gen;

@@ -5121,17 +5201,19 @@ static int changed_inode(struct send_ctx *sctx,
 			/*
 			 * First, process the inode as if it was deleted.
 			 */
-			sctx->cur_inode_gen = right_gen;
-			sctx->cur_inode_new = 0;
-			sctx->cur_inode_deleted = 1;
-			sctx->cur_inode_size = btrfs_inode_size(
+			if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+				sctx->cur_inode_gen = right_gen;
+				sctx->cur_inode_new = 0;
+				sctx->cur_inode_deleted = 1;
+				sctx->cur_inode_size = btrfs_inode_size(
 					sctx->right_path->nodes[0], right_ii);
-			sctx->cur_inode_mode = btrfs_inode_mode(
+				sctx->cur_inode_mode = btrfs_inode_mode(
 					sctx->right_path->nodes[0], right_ii);
-			ret = process_all_refs(sctx,
-					BTRFS_COMPARE_TREE_DELETED);
-			if (ret < 0)
-				goto out;
+				ret = process_all_refs(sctx,
+						   BTRFS_COMPARE_TREE_DELETED);
+				if (ret < 0)
+					goto out;
+			}
 
 			/*
 			 * Now process the inode as if it was new.

@@ -5145,29 +5227,38 @@ static int changed_inode(struct send_ctx *sctx,
 					sctx->left_path->nodes[0], left_ii);
 			sctx->cur_inode_rdev = btrfs_inode_rdev(
 					sctx->left_path->nodes[0], left_ii);
-			ret = send_create_inode_if_needed(sctx);
-			if (ret < 0)
-				goto out;
-
-			ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
-			if (ret < 0)
-				goto out;
+			if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+				ret = send_create_inode_if_needed(sctx);
+				if (ret < 0)
+					goto out;
+				ret = process_all_refs(sctx,
+						       BTRFS_COMPARE_TREE_NEW);
+				if (ret < 0)
+					goto out;
+			}
 			/*
 			 * Advance send_progress now as we did not get into
 			 * process_recorded_refs_if_needed in the new_gen case.
 			 */
 			sctx->send_progress = sctx->cur_ino + 1;
 
-			/*
-			 * Now process all extents and xattrs of the inode as if
-			 * they were all new.
-			 */
-			ret = process_all_extents(sctx);
-			if (ret < 0)
-				goto out;
-			ret = process_all_new_xattrs(sctx);
-			if (ret < 0)
-				goto out;
+			if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+				if (S_ISREG(sctx->cur_inode_mode))
+					sctx->total_data_size +=
+						sctx->cur_inode_size;
+				/* TODO: maybe account for xattrs one day too */
+			} else {
+				/*
+				 * Now process all extents and xattrs of the
+				 * inode as if they were all new.
+				 */
+				ret = process_all_extents(sctx);
+				if (ret < 0)
+					goto out;
+				ret = process_all_new_xattrs(sctx);
+				if (ret < 0)
+					goto out;
+			}
 		} else {
 			sctx->cur_inode_gen = left_gen;
 			sctx->cur_inode_new = 0;

@@ -5201,6 +5292,9 @@ static int changed_ref(struct send_ctx *sctx,
 
 	BUG_ON(sctx->cur_ino != sctx->cmp_key->objectid);
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 	if (!sctx->cur_inode_new_gen &&
 	    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
 		if (result == BTRFS_COMPARE_TREE_NEW)

@@ -5226,6 +5320,9 @@ static int changed_xattr(struct send_ctx *sctx,
 
 	BUG_ON(sctx->cur_ino != sctx->cmp_key->objectid);
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
 		if (result == BTRFS_COMPARE_TREE_NEW)
 			ret = process_new_xattr(sctx);

@@ -5335,6 +5432,8 @@ static int changed_cb(struct btrfs_root *left_root,
 	if (result == BTRFS_COMPARE_TREE_SAME) {
 		if (key->type == BTRFS_INODE_REF_KEY ||
 		    key->type == BTRFS_INODE_EXTREF_KEY) {
+			if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+				return 0;
 			ret = compare_refs(sctx, left_path, key);
 			if (!ret)
 				return 0;

@@ -5431,6 +5530,24 @@ out:
 	return ret;
 }
 
+static int compute_total_data_size(struct send_ctx *sctx)
+{
+	int ret;
+
+	sctx->total_data_size = 0;
+
+	if (sctx->parent_root) {
+		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root,
+					  changed_cb, sctx);
+		if (!ret)
+			ret = finish_inode_if_needed(sctx, 1);
+	} else {
+		ret = full_send_tree(sctx);
+	}
+
+	return ret;
+}
+
 static int send_subvol(struct send_ctx *sctx)
 {
 	int ret;

@@ -5445,6 +5562,19 @@ static int send_subvol(struct send_ctx *sctx)
 	if (ret < 0)
 		goto out;
 
+	if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE) {
+		sctx->phase = SEND_PHASE_COMPUTE_DATA_SIZE;
+		ret = compute_total_data_size(sctx);
+		if (ret < 0)
+			goto out;
+		ret = send_total_data_size(sctx, sctx->total_data_size);
+		if (ret < 0)
+			goto out;
+		sctx->phase = SEND_PHASE_STREAM_CHANGES;
+		sctx->cur_ino = 0;
+		sctx->send_progress = 0;
+	}
+
 	if (sctx->parent_root) {
 		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root,
 				changed_cb, sctx);

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-16 13:49 UTC #2 lore raw

PATCH v2 2/4 Btrfs: send, implement total data size command to allow for progress estimation

After modifying btrfs-progs to understand this new command and report progress, here's an example (the -o flag tells btrfs send to pass the new flag to the kernel's send ioctl):

    $ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_base
    At subvol snap_base
    About to receive 9212392667 bytes
    Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s

    $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
    At subvol /mnt/sdd/snap_incr
    At subvol snap_incr
    About to receive 9571342213 bytes
    Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/s

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c | 194 ++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 162 insertions(+), 32 deletions(-)

fs/btrfs/send.c +162 ~~-32~~

index 53712aa..f5db492 100644
@@ -81,7 +81,13 @@ struct clone_root {
 #define SEND_CTX_MAX_NAME_CACHE_SIZE 128
 #define SEND_CTX_NAME_CACHE_CLEAN_SIZE (SEND_CTX_MAX_NAME_CACHE_SIZE * 2)
 
+enum btrfs_send_phase {
+	SEND_PHASE_STREAM_CHANGES,
+	SEND_PHASE_COMPUTE_DATA_SIZE,
+};
+
 struct send_ctx {
+	enum btrfs_send_phase phase;
 	struct file *send_filp;
 	loff_t send_off;
 	char *send_buf;

@@ -116,6 +122,7 @@ struct send_ctx {
 	u64 cur_inode_last_extent;
 
 	u64 send_progress;
+	u64 total_data_size;
 
 	struct list_head new_refs;
 	struct list_head deleted_refs;

@@ -692,6 +699,8 @@ static int send_rename(struct send_ctx *sctx,
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_rename %s -> %s\n", from->start, to->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME);

@@ -716,6 +725,8 @@ static int send_link(struct send_ctx *sctx,
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_link %s -> %s\n", path->start, lnk->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_LINK);

@@ -739,6 +750,8 @@ static int send_unlink(struct send_ctx *sctx, struct fs_path *path)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_unlink %s\n", path->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK);

@@ -761,6 +774,8 @@ static int send_rmdir(struct send_ctx *sctx, struct fs_path *path)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_rmdir %s\n", path->start);
 
 	ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR);

@@ -2308,6 +2323,9 @@ static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size)
 	int ret = 0;
 	struct fs_path *p;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 verbose_printk("btrfs: send_truncate %llu size=%llu\n", ino, size);
 
 	p = fs_path_alloc();

@@ -2337,6 +2355,8 @@ static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode)
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_chmod %llu mode=%llu\n", ino, mode);
 
 	p = fs_path_alloc();

@@ -2366,6 +2386,8 @@ static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid)
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_chown %llu uid=%llu, gid=%llu\n", ino, uid, gid);
 
 	p = fs_path_alloc();

@@ -2401,6 +2423,8 @@ static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen)
 	struct btrfs_key key;
 	int slot;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_utimes %llu\n", ino);
 
 	p = fs_path_alloc();

@@ -2463,6 +2487,8 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino)
 	u64 mode;
 	u64 rdev;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_create_inode %llu\n", ino);
 
 	p = fs_path_alloc();

@@ -2610,6 +2636,8 @@ static int send_create_inode_if_needed(struct send_ctx *sctx)
 {
 	int ret;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	if (S_ISDIR(sctx->cur_inode_mode)) {
 		ret = did_create_dir(sctx, sctx->cur_ino);
 		if (ret < 0)

@@ -2715,6 +2743,8 @@ static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen,
 	int ret;
 	struct fs_path *orphan;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	orphan = fs_path_alloc();
 	if (!orphan)
 		return -ENOMEM;

@@ -3083,6 +3113,8 @@ static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm)
 	int ret;
 	u64 ancestor = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	name = fs_path_alloc();
 	from_path = fs_path_alloc();
 	if (!name || !from_path) {

@@ -3337,6 +3369,9 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move)
 	int is_orphan = 0;
 	u64 last_dir_ino_rm = 0;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 verbose_printk("btrfs: process_recorded_refs %llu\n", sctx->cur_ino);
 
 	/*

@@ -3845,6 +3880,8 @@ static int process_all_refs(struct send_ctx *sctx,
 	iterate_inode_ref_t cb;
 	int pending_move = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	path = alloc_path_for_send();
 	if (!path)
 		return -ENOMEM;

@@ -4164,6 +4201,8 @@ static int process_all_new_xattrs(struct send_ctx *sctx)
 	struct extent_buffer *eb;
 	int slot;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	path = alloc_path_for_send();
 	if (!path)
 		return -ENOMEM;

@@ -4294,6 +4333,8 @@ static int send_write(struct send_ctx *sctx, u64 offset, u32 len)
 	struct fs_path *p;
 	ssize_t num_read = 0;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4329,6 +4370,22 @@ out:
 	return num_read;
 }
 
+static int send_total_data_size(struct send_ctx *sctx, u64 data_size)
+{
+	int ret;
+
+	ret = begin_cmd(sctx, BTRFS_SEND_C_TOTAL_DATA_SIZE);
+	if (ret < 0)
+		goto out;
+
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, data_size);
+	ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+	return ret;
+}
+
 /*
  * Send a clone command to user space.
  */

@@ -4340,6 +4397,8 @@ static int send_clone(struct send_ctx *sctx,
 	struct fs_path *p;
 	u64 gen;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 verbose_printk("btrfs: send_clone offset=%llu, len=%d, clone_root=%llu, "
 	       "clone_inode=%llu, clone_offset=%llu\n", offset, len,
 		clone_root->root->objectid, clone_root->ino,

@@ -4398,6 +4457,8 @@ static int send_update_extent(struct send_ctx *sctx,
 	int ret = 0;
 	struct fs_path *p;
 
+	ASSERT(sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE);
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4429,6 +4490,11 @@ static int send_hole(struct send_ctx *sctx, u64 end)
 	u64 len;
 	int ret = 0;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		sctx->total_data_size += end - offset;
+		return 0;
+	}
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4492,6 +4558,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		goto out;
 	}
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		if (offset < sctx->cur_inode_size)
+			sctx->total_data_size += len;
+		goto out;
+	}
+
 	if (clone_root && IS_ALIGNED(offset + len, bs)) {
 		ret = send_clone(sctx, offset, len, clone_root);
 	} else if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) {

@@ -4825,10 +4897,12 @@ static int process_extent(struct send_ctx *sctx,
 		}
 	}
 
-	ret = find_extent_clone(sctx, path, key->objectid, key->offset,
-			sctx->cur_inode_size, &found_clone);
-	if (ret != -ENOENT && ret < 0)
-		goto out;
+	if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+		ret = find_extent_clone(sctx, path, key->objectid, key->offset,
+					sctx->cur_inode_size, &found_clone);
+		if (ret != -ENOENT && ret < 0)
+			goto out;
+	}
 
 	ret = send_write_or_clone(sctx, path, key, found_clone);
 	if (ret)

@@ -4958,6 +5032,9 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 	if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino)
 		goto out;
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		goto truncate_inode;
+
 	ret = get_inode_info(sctx->send_root, sctx->cur_ino, NULL, NULL,
 			&left_mode, &left_uid, &left_gid, NULL);
 	if (ret < 0)

@@ -4980,6 +5057,7 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 			need_chmod = 1;
 	}
 
+truncate_inode:
 	if (S_ISREG(sctx->cur_inode_mode)) {
 		if (need_send_hole(sctx)) {
 			if (sctx->cur_inode_last_extent == (u64)-1 ||

@@ -5019,7 +5097,8 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end)
 	 * If other directory inodes depended on our current directory
 	 * inode's move/rename, now do their move/rename operations.
 	 */
-	if (!is_waiting_for_move(sctx, sctx->cur_ino)) {
+	if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE &&
+	    !is_waiting_for_move(sctx, sctx->cur_ino)) {
 		ret = apply_children_dir_moves(sctx);
 		if (ret)
 			goto out;

@@ -5103,7 +5182,8 @@ static int changed_inode(struct send_ctx *sctx,
 				sctx->left_path->nodes[0], left_ii);
 		sctx->cur_inode_rdev = btrfs_inode_rdev(
 				sctx->left_path->nodes[0], left_ii);
-		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID)
+		if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID &&
+		    sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE)
 			ret = send_create_inode_if_needed(sctx);
 	} else if (result == BTRFS_COMPARE_TREE_DELETED) {
 		sctx->cur_inode_gen = right_gen;

@@ -5125,17 +5205,19 @@ static int changed_inode(struct send_ctx *sctx,
 			/*
 			 * First, process the inode as if it was deleted.
 			 */
-			sctx->cur_inode_gen = right_gen;
-			sctx->cur_inode_new = 0;
-			sctx->cur_inode_deleted = 1;
-			sctx->cur_inode_size = btrfs_inode_size(
+			if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+				sctx->cur_inode_gen = right_gen;
+				sctx->cur_inode_new = 0;
+				sctx->cur_inode_deleted = 1;
+				sctx->cur_inode_size = btrfs_inode_size(
 					sctx->right_path->nodes[0], right_ii);
-			sctx->cur_inode_mode = btrfs_inode_mode(
+				sctx->cur_inode_mode = btrfs_inode_mode(
 					sctx->right_path->nodes[0], right_ii);
-			ret = process_all_refs(sctx,
-					BTRFS_COMPARE_TREE_DELETED);
-			if (ret < 0)
-				goto out;
+				ret = process_all_refs(sctx,
+						   BTRFS_COMPARE_TREE_DELETED);
+				if (ret < 0)
+					goto out;
+			}
 
 			/*
 			 * Now process the inode as if it was new.

@@ -5149,29 +5231,38 @@ static int changed_inode(struct send_ctx *sctx,
 					sctx->left_path->nodes[0], left_ii);
 			sctx->cur_inode_rdev = btrfs_inode_rdev(
 					sctx->left_path->nodes[0], left_ii);
-			ret = send_create_inode_if_needed(sctx);
-			if (ret < 0)
-				goto out;
-
-			ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW);
-			if (ret < 0)
-				goto out;
+			if (sctx->phase != SEND_PHASE_COMPUTE_DATA_SIZE) {
+				ret = send_create_inode_if_needed(sctx);
+				if (ret < 0)
+					goto out;
+				ret = process_all_refs(sctx,
+						       BTRFS_COMPARE_TREE_NEW);
+				if (ret < 0)
+					goto out;
+			}
 			/*
 			 * Advance send_progress now as we did not get into
 			 * process_recorded_refs_if_needed in the new_gen case.
 			 */
 			sctx->send_progress = sctx->cur_ino + 1;
 
-			/*
-			 * Now process all extents and xattrs of the inode as if
-			 * they were all new.
-			 */
-			ret = process_all_extents(sctx);
-			if (ret < 0)
-				goto out;
-			ret = process_all_new_xattrs(sctx);
-			if (ret < 0)
-				goto out;
+			if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+				if (S_ISREG(sctx->cur_inode_mode))
+					sctx->total_data_size +=
+						sctx->cur_inode_size;
+				/* TODO: maybe account for xattrs one day too */
+			} else {
+				/*
+				 * Now process all extents and xattrs of the
+				 * inode as if they were all new.
+				 */
+				ret = process_all_extents(sctx);
+				if (ret < 0)
+					goto out;
+				ret = process_all_new_xattrs(sctx);
+				if (ret < 0)
+					goto out;
+			}
 		} else {
 			sctx->cur_inode_gen = left_gen;
 			sctx->cur_inode_new = 0;

@@ -5205,6 +5296,9 @@ static int changed_ref(struct send_ctx *sctx,
 
 	BUG_ON(sctx->cur_ino != sctx->cmp_key->objectid);
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 	if (!sctx->cur_inode_new_gen &&
 	    sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) {
 		if (result == BTRFS_COMPARE_TREE_NEW)

@@ -5230,6 +5324,9 @@ static int changed_xattr(struct send_ctx *sctx,
 
 	BUG_ON(sctx->cur_ino != sctx->cmp_key->objectid);
 
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+		return 0;
+
 	if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) {
 		if (result == BTRFS_COMPARE_TREE_NEW)
 			ret = process_new_xattr(sctx);

@@ -5339,6 +5436,8 @@ static int changed_cb(struct btrfs_root *left_root,
 	if (result == BTRFS_COMPARE_TREE_SAME) {
 		if (key->type == BTRFS_INODE_REF_KEY ||
 		    key->type == BTRFS_INODE_EXTREF_KEY) {
+			if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE)
+				return 0;
 			ret = compare_refs(sctx, left_path, key);
 			if (!ret)
 				return 0;

@@ -5435,6 +5534,24 @@ out:
 	return ret;
 }
 
+static int compute_total_data_size(struct send_ctx *sctx)
+{
+	int ret;
+
+	sctx->total_data_size = 0;
+
+	if (sctx->parent_root) {
+		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root,
+					  changed_cb, sctx);
+		if (!ret)
+			ret = finish_inode_if_needed(sctx, 1);
+	} else {
+		ret = full_send_tree(sctx);
+	}
+
+	return ret;
+}
+
 static int send_subvol(struct send_ctx *sctx)
 {
 	int ret;

@@ -5449,6 +5566,19 @@ static int send_subvol(struct send_ctx *sctx)
 	if (ret < 0)
 		goto out;
 
+	if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE) {
+		sctx->phase = SEND_PHASE_COMPUTE_DATA_SIZE;
+		ret = compute_total_data_size(sctx);
+		if (ret < 0)
+			goto out;
+		ret = send_total_data_size(sctx, sctx->total_data_size);
+		if (ret < 0)
+			goto out;
+		sctx->phase = SEND_PHASE_STREAM_CHANGES;
+		sctx->cur_ino = 0;
+		sctx->send_progress = 0;
+	}
+
 	if (sctx->parent_root) {
 		ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root,
 				changed_cb, sctx);

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-15 15:40 UTC #3 lore raw

PATCH 3/4 Btrfs: send, use fallocate command to punch holes

In reply to Filipe David Borba Manana

Instead of sending a write command with a data buffer filled with 0 value bytes, use the fallocate command, introduced in the send stream version 2, to tell the receiver to punch a file hole using the fallocate system call.

Signed-off-by Filipe David Borba Manana <[email protected]>

---
 fs/btrfs/send.c | 51 +++++++++++++++++++++++++++++++--------------------
 fs/btrfs/send.h |  4 ++++
 2 files changed, 35 insertions(+), 20 deletions(-)

fs/btrfs/send.c +31 ~~-20~~

index 0cff503..26a09ba 100644
@@ -564,6 +564,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
 		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
 	}
 
+TLV_PUT_DEFINE_INT(32)
 TLV_PUT_DEFINE_INT(64)
 
 static int tlv_put_string(struct send_ctx *sctx, u16 attr,

@@ -4479,15 +4480,14 @@ out:
 	return ret;
 }
 
-static int send_hole(struct send_ctx *sctx, u64 end)
+static int send_fallocate(struct send_ctx *sctx, u32 flags,
+			  u64 offset, u64 len)
 {
 	struct fs_path *p = NULL;
-	u64 offset = sctx->cur_inode_last_extent;
-	u64 len;
 	int ret = 0;
 
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		sctx->total_data_size += end - offset;
+		sctx->total_data_size += len;
 		return 0;
 	}

@@ -4496,27 +4496,32 @@ static int send_hole(struct send_ctx *sctx, u64 end)
 		return -ENOMEM;
 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
 	if (ret < 0)
-		goto tlv_put_failure;
-	memset(sctx->read_buf, 0, BTRFS_SEND_READ_SIZE);
-	while (offset < end) {
-		len = min_t(u64, end - offset, BTRFS_SEND_READ_SIZE);
+		goto out;
+
+	ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
+	if (ret < 0)
+		goto out;
+	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
+	TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, flags);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
+	ret = send_cmd(sctx);
 
-		ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
-		if (ret < 0)
-			break;
-		TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
-		TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
-		TLV_PUT(sctx, BTRFS_SEND_A_DATA, sctx->read_buf, len);
-		ret = send_cmd(sctx);
-		if (ret < 0)
-			break;
-		offset += len;
-	}
 tlv_put_failure:
+out:
 	fs_path_free(p);
 	return ret;
 }
 
+static int send_hole(struct send_ctx *sctx, u64 end)
+{
+	u64 offset = sctx->cur_inode_last_extent;
+	u64 len = end - offset;
+
+	return send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+			      offset, len);
+}
+
 static int send_write_or_clone(struct send_ctx *sctx,
 			       struct btrfs_path *path,
 			       struct btrfs_key *key,

@@ -4547,7 +4552,8 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
 	}
 
-	if (offset + len > sctx->cur_inode_size)
+	if (offset < sctx->cur_inode_size &&
+	    offset + len > sctx->cur_inode_size)
 		len = sctx->cur_inode_size - offset;
 	if (len == 0) {
 		ret = 0;

@@ -4564,6 +4570,11 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		ret = send_clone(sctx, offset, len, clone_root);
 	} else if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) {
 		ret = send_update_extent(sctx, offset, len);
+	} else if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0 &&
+		   type != BTRFS_FILE_EXTENT_INLINE &&
+		   offset < sctx->cur_inode_size) {
+		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+				     offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

fs/btrfs/send.h +4 -0

index 7b5df08..a03f66c 100644
@@ -140,6 +140,10 @@ enum {
 #define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
 #define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
 
+#define BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS        \
+	(BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE |  \
+	 BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-16 13:50 UTC #4 lore raw

PATCH v2 3/4 Btrfs: send, use fallocate command to punch holes

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
 fs/btrfs/send.h |  4 ++++
 2 files changed, 55 insertions(+), 5 deletions(-)

fs/btrfs/send.c +51 -5

index f5db492..2c6d58c 100644
@@ -564,6 +564,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
 		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
 	}
 
+TLV_PUT_DEFINE_INT(32)
 TLV_PUT_DEFINE_INT(64)
 
 static int tlv_put_string(struct send_ctx *sctx, u16 attr,

@@ -4483,15 +4484,16 @@ out:
 	return ret;
 }
 
-static int send_hole(struct send_ctx *sctx, u64 end)
+static int send_fallocate(struct send_ctx *sctx, u32 flags,
+			  u64 offset, u64 len)
 {
 	struct fs_path *p = NULL;
-	u64 offset = sctx->cur_inode_last_extent;
-	u64 len;
 	int ret = 0;
 
+	ASSERT(sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE);
+
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		sctx->total_data_size += end - offset;
+		sctx->total_data_size += len;
 		return 0;
 	}

@@ -4500,6 +4502,43 @@ static int send_hole(struct send_ctx *sctx, u64 end)
 		return -ENOMEM;
 	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
 	if (ret < 0)
+		goto out;
+
+	ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
+	if (ret < 0)
+		goto out;
+	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
+	TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, flags);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
+	ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+	fs_path_free(p);
+	return ret;
+}
+
+static int send_hole(struct send_ctx *sctx, u64 end)
+{
+	struct fs_path *p = NULL;
+	u64 offset = sctx->cur_inode_last_extent;
+	u64 len = end - offset;
+	int ret = 0;
+
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		sctx->total_data_size += len;
+		return 0;
+	}
+
+	if (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)
+		return send_fallocate(sctx,
+				      BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+				      offset,
+				      len);
+
+	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
+	if (ret < 0)
 		goto tlv_put_failure;
 	memset(sctx->read_buf, 0, BTRFS_SEND_READ_SIZE);
 	while (offset < end) {

@@ -4551,7 +4590,8 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
 	}
 
-	if (offset + len > sctx->cur_inode_size)
+	if (offset < sctx->cur_inode_size &&
+	    offset + len > sctx->cur_inode_size)
 		len = sctx->cur_inode_size - offset;
 	if (len == 0) {
 		ret = 0;

@@ -4568,6 +4608,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		ret = send_clone(sctx, offset, len, clone_root);
 	} else if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) {
 		ret = send_update_extent(sctx, offset, len);
+	} else if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0 &&
+		   type != BTRFS_FILE_EXTENT_INLINE &&
+		   (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE) &&
+		   offset < sctx->cur_inode_size) {
+		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+				     offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

fs/btrfs/send.h +4 -0

index 367030d..a632c0d 100644
@@ -141,6 +141,10 @@ enum {
 #define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
 #define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
 
+#define BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS        \
+	(BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE |  \
+	 BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-16 15:16 UTC #5 lore raw

PATCH v3 3/4 Btrfs: send, use fallocate command to punch holes

In reply to Filipe David Borba Manana

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
 fs/btrfs/send.h |  4 ++++
 2 files changed, 56 insertions(+), 3 deletions(-)

fs/btrfs/send.c +52 -3

index f5db492..bb9afea 100644
@@ -564,6 +564,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len)
 		return tlv_put(sctx, attr, &__tmp, sizeof(__tmp));	\
 	}
 
+TLV_PUT_DEFINE_INT(32)
 TLV_PUT_DEFINE_INT(64)
 
 static int tlv_put_string(struct send_ctx *sctx, u16 attr,

@@ -4483,18 +4484,59 @@ out:
 	return ret;
 }
 
+static int send_fallocate(struct send_ctx *sctx, u32 flags,
+			  u64 offset, u64 len)
+{
+	struct fs_path *p = NULL;
+	int ret = 0;
+
+	ASSERT(sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE);
+
+	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
+		sctx->total_data_size += len;
+		return 0;
+	}
+
+	p = fs_path_alloc();
+	if (!p)
+		return -ENOMEM;
+	ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
+	if (ret < 0)
+		goto out;
+
+	ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE);
+	if (ret < 0)
+		goto out;
+	TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
+	TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_FLAGS, flags);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
+	TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len);
+	ret = send_cmd(sctx);
+
+tlv_put_failure:
+out:
+	fs_path_free(p);
+	return ret;
+}
+
 static int send_hole(struct send_ctx *sctx, u64 end)
 {
 	struct fs_path *p = NULL;
 	u64 offset = sctx->cur_inode_last_extent;
-	u64 len;
+	u64 len = end - offset;
 	int ret = 0;
 
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		sctx->total_data_size += end - offset;
+		sctx->total_data_size += len;
 		return 0;
 	}
 
+	if (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)
+		return send_fallocate(sctx,
+				      BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+				      offset,
+				      len);
+
 	p = fs_path_alloc();
 	if (!p)
 		return -ENOMEM;

@@ -4551,7 +4593,8 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
 	}
 
-	if (offset + len > sctx->cur_inode_size)
+	if (offset < sctx->cur_inode_size &&
+	    offset + len > sctx->cur_inode_size)
 		len = sctx->cur_inode_size - offset;
 	if (len == 0) {
 		ret = 0;

@@ -4568,6 +4611,12 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		ret = send_clone(sctx, offset, len, clone_root);
 	} else if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) {
 		ret = send_update_extent(sctx, offset, len);
+	} else if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0 &&
+		   type != BTRFS_FILE_EXTENT_INLINE &&
+		   (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE) &&
+		   offset < sctx->cur_inode_size) {
+		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+				     offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

fs/btrfs/send.h +4 -0

index 367030d..a632c0d 100644
@@ -141,6 +141,10 @@ enum {
 #define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
 #define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
 
+#define BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS        \
+	(BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE |  \
+	 BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-15 15:40 UTC #6 lore raw

PATCH 4/4 Btrfs: send, use fallocate command to allocate extents

In reply to Filipe David Borba Manana

The send stream version 2 adds the fallocate command, which can be used to allocate extents for a file or punch holes in a file. Previously we were ignoring file prealloc extents or treating them as extents filled with 0 bytes and sending a regular write command to the stream.

After this change, together with my previous change titled:

"Btrfs: send, use fallocate command to punch holes"

an incremental send preserves the hole and data structure of files, which can be seen via calls to lseek with the whence parameter set to SEEK_DATA or SEEK_HOLE, as the example below shows:

mkfs.btrfs -f /dev/sdc
mount /dev/sdc /mnt
xfs_io -f -c "pwrite -S 0x01 -b 300000 0 300000" /mnt/foo btrfs subvolume snapshot -r /mnt /mnt/mysnap1

xfs_io -c "fpunch 100000 50000" /mnt/foo xfs_io -c "falloc 100000 50000" /mnt/foo xfs_io -c "pwrite -S 0xff -b 1000 120000 1000" /mnt/foo xfs_io -c "fpunch 250000 20000" /mnt/foo

# prealloc extents that start beyond the inode's size xfs_io -c "falloc -k 300000 1000000" /mnt/foo xfs_io -c "falloc -k 9000000 2000000" /mnt/foo

btrfs subvolume snapshot -r /mnt /mnt/mysnap2

btrfs send /mnt/mysnap1 -f /tmp/1.snap
btrfs send -p /mnt/mysnap1 /mnt/mysnap2 -f /tmp/2.snap

mkfs.btrfs -f /dev/sdd
mount /dev/sdd /mnt2
btrfs receive /mnt2 -f /tmp/1.snap
btrfs receive /mnt2 -f /tmp/2.snap

Before this change the hole/data structure differed between both filesystems:

    $ xfs_io -r -c 'seek -r -a 0' /mnt/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    102400
    DATA    118784
    HOLE    122880
    DATA    147456
    HOLE    253952
    DATA    266240
    HOLE    300000

    $ xfs_io -r -c 'seek -r -a 0' /mnt2/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    300000

After this change the second filesystem (/dev/sdd) ends up with the same hole/data structure as the first filesystem.

Also, after this change, prealloc extents that lie beyond the inode's size (were allocated with fallocate + keep size flag) are also replicated by an incremental send. For the above test, it can be observed via fiemap (or btrfs-debug-tree):

    $ xfs_io -r -c 'fiemap -l' /mnt2/mysnap2/foo
	0: [0..191]: 25096..25287 192 blocks
	1: [192..199]: 24672..24679 8 blocks
	2: [200..231]: 24584..24615 32 blocks
	3: [232..239]: 24680..24687 8 blocks
	4: [240..287]: 24616..24663 48 blocks
	5: [288..295]: 24688..24695 8 blocks
	6: [296..487]: 25392..25583 192 blocks
	7: [488..495]: 24696..24703 8 blocks
	8: [496..519]: hole 24 blocks
	9: [520..527]: 24704..24711 8 blocks
	10: [528..583]: 25624..25679 56 blocks
	11: [584..591]: 24712..24719 8 blocks
	12: [592..2543]: 26192..28143 1952 blocks
	13: [2544..17575]: hole 15032 blocks
	14: [17576..21487]: 28144..32055 3912 blocks

A test case for xfstests will follow.

Signed-off-by Filipe David Borba Manana <[email protected]>

---
 fs/btrfs/send.c | 63 ++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 42 insertions(+), 21 deletions(-)

fs/btrfs/send.c +42 ~~-21~~

index 26a09ba..5f1bbc0 100644
@@ -114,8 +114,9 @@ struct send_ctx {
 	u64 cur_ino;
 	u64 cur_inode_gen;
 	int cur_inode_new;
-	int cur_inode_new_gen;
-	int cur_inode_deleted;
+	int cur_inode_new_gen:1;
+	int cur_inode_skip_truncate:1;
+	int cur_inode_deleted:1;
 	u64 cur_inode_size;
 	u64 cur_inode_mode;
 	u64 cur_inode_rdev;

@@ -4561,8 +4562,7 @@ static int send_write_or_clone(struct send_ctx *sctx,
 	}
 
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		if (offset < sctx->cur_inode_size)
-			sctx->total_data_size += len;
+		sctx->total_data_size += len;
 		goto out;
 	}

@@ -4575,6 +4575,26 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		   offset < sctx->cur_inode_size) {
 		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
 				     offset, len);
+	} else if (type == BTRFS_FILE_EXTENT_PREALLOC) {
+		u32 mode = 0;
+		if (offset < sctx->cur_inode_size) {
+			ret = send_fallocate(sctx,
+					     BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+					     offset, len);
+			if (ret)
+				goto out;
+		} else {
+			if (!sctx->cur_inode_skip_truncate) {
+				ret = send_truncate(sctx, sctx->cur_ino,
+						    sctx->cur_inode_gen,
+						    sctx->cur_inode_size);
+				if (ret < 0)
+					goto out;
+				sctx->cur_inode_skip_truncate = 1;
+			}
+			mode |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+		}
+		ret = send_fallocate(sctx, mode, offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

@@ -4883,19 +4903,16 @@ static int process_extent(struct send_ctx *sctx,
 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
 				    struct btrfs_file_extent_item);
 		type = btrfs_file_extent_type(path->nodes[0], ei);
-		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
-		    type == BTRFS_FILE_EXTENT_REG) {
-			/*
-			 * The send spec does not have a prealloc command yet,
-			 * so just leave a hole for prealloc'ed extents until
-			 * we have enough commands queued up to justify rev'ing
-			 * the send spec.
-			 */
-			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
-				ret = 0;
-				goto out;
-			}
-
+		if (type == BTRFS_FILE_EXTENT_PREALLOC) {
+			u64 len;
+			u32 mode = 0;
+
+			len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
+			if (key->offset >= sctx->cur_inode_size)
+				mode |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+			ret = send_fallocate(sctx, mode, key->offset, len);
+			goto out;
+		} else if (type == BTRFS_FILE_EXTENT_REG) {
 			/* Have a hole, just skip it. */
 			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
 				ret = 0;

@@ -5081,10 +5098,13 @@ truncate_inode:
 					goto out;
 			}
 		}
-		ret = send_truncate(sctx, sctx->cur_ino, sctx->cur_inode_gen,
-				sctx->cur_inode_size);
-		if (ret < 0)
-			goto out;
+		if (!sctx->cur_inode_skip_truncate) {
+			ret = send_truncate(sctx, sctx->cur_ino,
+					    sctx->cur_inode_gen,
+					    sctx->cur_inode_size);
+			if (ret < 0)
+				goto out;
+		}
 	}
 
 	if (need_chown) {

@@ -5139,6 +5159,7 @@ static int changed_inode(struct send_ctx *sctx,
 	sctx->cur_ino = key->objectid;
 	sctx->cur_inode_new_gen = 0;
 	sctx->cur_inode_last_extent = (u64)-1;
+	sctx->cur_inode_skip_truncate = 0;
 
 	/*
 	 * Set send_progress to current inode. This will tell all get_cur_xxx

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-16 13:52 UTC #7 lore raw

PATCH v2 4/4 Btrfs: send, use fallocate command to allocate extents

After this change, together with my previous change titled:

"Btrfs: send, use fallocate command to punch holes"

an incremental send preserves the hole and data structure of files, which can be seen via calls to lseek with the whence parameter set to SEEK_DATA or SEEK_HOLE, as the example below shows:

mkfs.btrfs -f /dev/sdc
mount /dev/sdc /mnt
xfs_io -f -c "pwrite -S 0x01 -b 300000 0 300000" /mnt/foo btrfs subvolume snapshot -r /mnt /mnt/mysnap1

xfs_io -c "fpunch 100000 50000" /mnt/foo xfs_io -c "falloc 100000 50000" /mnt/foo xfs_io -c "pwrite -S 0xff -b 1000 120000 1000" /mnt/foo xfs_io -c "fpunch 250000 20000" /mnt/foo

# prealloc extents that start beyond the inode's size xfs_io -c "falloc -k 300000 1000000" /mnt/foo xfs_io -c "falloc -k 9000000 2000000" /mnt/foo

btrfs subvolume snapshot -r /mnt /mnt/mysnap2

btrfs send /mnt/mysnap1 -f /tmp/1.snap
btrfs send -p /mnt/mysnap1 /mnt/mysnap2 -f /tmp/2.snap

mkfs.btrfs -f /dev/sdd
mount /dev/sdd /mnt2
btrfs receive /mnt2 -f /tmp/1.snap
btrfs receive /mnt2 -f /tmp/2.snap

Before this change the hole/data structure differed between both filesystems:

    $ xfs_io -r -c 'seek -r -a 0' /mnt/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    102400
    DATA    118784
    HOLE    122880
    DATA    147456
    HOLE    253952
    DATA    266240
    HOLE    300000

    $ xfs_io -r -c 'seek -r -a 0' /mnt2/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    300000

After this change the second filesystem (/dev/sdd) ends up with the same hole/data structure as the first filesystem.

    $ xfs_io -r -c 'fiemap -l' /mnt2/mysnap2/foo
	0: [0..191]: 25096..25287 192 blocks
	1: [192..199]: 24672..24679 8 blocks
	2: [200..231]: 24584..24615 32 blocks
	3: [232..239]: 24680..24687 8 blocks
	4: [240..287]: 24616..24663 48 blocks
	5: [288..295]: 24688..24695 8 blocks
	6: [296..487]: 25392..25583 192 blocks
	7: [488..495]: 24696..24703 8 blocks
	8: [496..519]: hole 24 blocks
	9: [520..527]: 24704..24711 8 blocks
	10: [528..583]: 25624..25679 56 blocks
	11: [584..591]: 24712..24719 8 blocks
	12: [592..2543]: 26192..28143 1952 blocks
	13: [2544..17575]: hole 15032 blocks
	14: [17576..21487]: 28144..32055 3912 blocks

A test case for xfstests will follow.

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c | 70 +++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 22 deletions(-)

fs/btrfs/send.c +48 ~~-22~~

index 2c6d58c..043fd43 100644
@@ -113,9 +113,10 @@ struct send_ctx {
 	 */
 	u64 cur_ino;
 	u64 cur_inode_gen;
-	int cur_inode_new;
-	int cur_inode_new_gen;
-	int cur_inode_deleted;
+	u8 cur_inode_new:1;
+	u8 cur_inode_new_gen:1;
+	u8 cur_inode_skip_truncate:1;
+	u8 cur_inode_deleted:1;
 	u64 cur_inode_size;
 	u64 cur_inode_mode;
 	u64 cur_inode_rdev;

@@ -4599,8 +4600,7 @@ static int send_write_or_clone(struct send_ctx *sctx,
 	}
 
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		if (offset < sctx->cur_inode_size)
-			sctx->total_data_size += len;
+		sctx->total_data_size += len;
 		goto out;
 	}

@@ -4614,6 +4614,27 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		   offset < sctx->cur_inode_size) {
 		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
 				     offset, len);
+	} else if (type == BTRFS_FILE_EXTENT_PREALLOC &&
+		   (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)) {
+		u32 mode = 0;
+		if (offset < sctx->cur_inode_size) {
+			ret = send_fallocate(sctx,
+					     BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+					     offset, len);
+			if (ret)
+				goto out;
+		} else {
+			if (!sctx->cur_inode_skip_truncate) {
+				ret = send_truncate(sctx, sctx->cur_ino,
+						    sctx->cur_inode_gen,
+						    sctx->cur_inode_size);
+				if (ret < 0)
+					goto out;
+				sctx->cur_inode_skip_truncate = 1;
+			}
+			mode |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+		}
+		ret = send_fallocate(sctx, mode, offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

@@ -4922,19 +4943,20 @@ static int process_extent(struct send_ctx *sctx,
 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
 				    struct btrfs_file_extent_item);
 		type = btrfs_file_extent_type(path->nodes[0], ei);
-		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
-		    type == BTRFS_FILE_EXTENT_REG) {
-			/*
-			 * The send spec does not have a prealloc command yet,
-			 * so just leave a hole for prealloc'ed extents until
-			 * we have enough commands queued up to justify rev'ing
-			 * the send spec.
-			 */
-			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
-				ret = 0;
-				goto out;
-			}
-
+		if (type == BTRFS_FILE_EXTENT_PREALLOC &&
+		    (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)) {
+			u64 len;
+			u32 flags = 0;
+
+			len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
+			if (key->offset >= sctx->cur_inode_size)
+				flags |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+			ret = send_fallocate(sctx, flags, key->offset, len);
+			goto out;
+		} else if (type == BTRFS_FILE_EXTENT_PREALLOC) {
+			ret = 0;
+			goto out;
+		} else if (type == BTRFS_FILE_EXTENT_REG) {
 			/* Have a hole, just skip it. */
 			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
 				ret = 0;

@@ -5120,10 +5142,13 @@ truncate_inode:
 					goto out;
 			}
 		}
-		ret = send_truncate(sctx, sctx->cur_ino, sctx->cur_inode_gen,
-				sctx->cur_inode_size);
-		if (ret < 0)
-			goto out;
+		if (!sctx->cur_inode_skip_truncate) {
+			ret = send_truncate(sctx, sctx->cur_ino,
+					    sctx->cur_inode_gen,
+					    sctx->cur_inode_size);
+			if (ret < 0)
+				goto out;
+		}
 	}
 
 	if (need_chown) {

@@ -5178,6 +5203,7 @@ static int changed_inode(struct send_ctx *sctx,
 	sctx->cur_ino = key->objectid;
 	sctx->cur_inode_new_gen = 0;
 	sctx->cur_inode_last_extent = (u64)-1;
+	sctx->cur_inode_skip_truncate = 0;
 
 	/*
 	 * Set send_progress to current inode. This will tell all get_cur_xxx

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-16 16:56 UTC #8 lore raw

PATCH v3 4/4 Btrfs: send, use fallocate command to allocate extents

In reply to Filipe David Borba Manana

After this change, together with my previous change titled:

"Btrfs: send, use fallocate command to punch holes"

an incremental send preserves the hole and data structure of files, which can be seen via calls to lseek with the whence parameter set to SEEK_DATA or SEEK_HOLE, as the example below shows:

mkfs.btrfs -f /dev/sdc
mount /dev/sdc /mnt
xfs_io -f -c "pwrite -S 0x01 -b 300000 0 300000" /mnt/foo btrfs subvolume snapshot -r /mnt /mnt/mysnap1

xfs_io -c "fpunch 100000 50000" /mnt/foo xfs_io -c "falloc 100000 50000" /mnt/foo xfs_io -c "pwrite -S 0xff -b 1000 120000 1000" /mnt/foo xfs_io -c "fpunch 250000 20000" /mnt/foo

# prealloc extents that start beyond the inode's size xfs_io -c "falloc -k 300000 1000000" /mnt/foo xfs_io -c "falloc -k 9000000 2000000" /mnt/foo

btrfs subvolume snapshot -r /mnt /mnt/mysnap2

btrfs send /mnt/mysnap1 -f /tmp/1.snap
btrfs send -p /mnt/mysnap1 /mnt/mysnap2 -f /tmp/2.snap

mkfs.btrfs -f /dev/sdd
mount /dev/sdd /mnt2
btrfs receive /mnt2 -f /tmp/1.snap
btrfs receive /mnt2 -f /tmp/2.snap

Before this change the hole/data structure differed between both filesystems:

    $ xfs_io -r -c 'seek -r -a 0' /mnt/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    102400
    DATA    118784
    HOLE    122880
    DATA    147456
    HOLE    253952
    DATA    266240
    HOLE    300000

    $ xfs_io -r -c 'seek -r -a 0' /mnt2/mysnap2/foo
    Whence  Result
    DATA    0
    HOLE    300000

After this change the second filesystem (/dev/sdd) ends up with the same hole/data structure as the first filesystem.

    $ xfs_io -r -c 'fiemap -l' /mnt2/mysnap2/foo
	0: [0..191]: 25096..25287 192 blocks
	1: [192..199]: 24672..24679 8 blocks
	2: [200..231]: 24584..24615 32 blocks
	3: [232..239]: 24680..24687 8 blocks
	4: [240..287]: 24616..24663 48 blocks
	5: [288..295]: 24688..24695 8 blocks
	6: [296..487]: 25392..25583 192 blocks
	7: [488..495]: 24696..24703 8 blocks
	8: [496..519]: hole 24 blocks
	9: [520..527]: 24704..24711 8 blocks
	10: [528..583]: 25624..25679 56 blocks
	11: [584..591]: 24712..24719 8 blocks
	12: [592..2543]: 26192..28143 1952 blocks
	13: [2544..17575]: hole 15032 blocks
	14: [17576..21487]: 28144..32055 3912 blocks

A test case for xfstests will follow.

Signed-off-by Filipe David Borba Manana <[email protected]>

---

V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Fixed rebase, removed some duplicate logic on truncate + falloc -k.

 fs/btrfs/send.c | 78 +++++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 21 deletions(-)

fs/btrfs/send.c +57 ~~-21~~

index bb9afea..86b6a87 100644
@@ -113,9 +113,10 @@ struct send_ctx {
 	 */
 	u64 cur_ino;
 	u64 cur_inode_gen;
-	int cur_inode_new;
-	int cur_inode_new_gen;
-	int cur_inode_deleted;
+	u8 cur_inode_new:1;
+	u8 cur_inode_new_gen:1;
+	u8 cur_inode_skip_truncate:1;
+	u8 cur_inode_deleted:1;
 	u64 cur_inode_size;
 	u64 cur_inode_mode;
 	u64 cur_inode_rdev;

@@ -4563,6 +4564,19 @@ tlv_put_failure:
 	return ret;
 }
 
+static int truncate_before_falloc(struct send_ctx *sctx)
+{
+	int ret = 0;
+
+	if (!sctx->cur_inode_skip_truncate) {
+		ret = send_truncate(sctx, sctx->cur_ino,
+				    sctx->cur_inode_gen,
+				    sctx->cur_inode_size);
+		sctx->cur_inode_skip_truncate = 1;
+	}
+	return ret;
+}
+
 static int send_write_or_clone(struct send_ctx *sctx,
 			       struct btrfs_path *path,
 			       struct btrfs_key *key,

@@ -4602,8 +4616,7 @@ static int send_write_or_clone(struct send_ctx *sctx,
 	}
 
 	if (sctx->phase == SEND_PHASE_COMPUTE_DATA_SIZE) {
-		if (offset < sctx->cur_inode_size)
-			sctx->total_data_size += len;
+		sctx->total_data_size += len;
 		goto out;
 	}

@@ -4617,6 +4630,20 @@ static int send_write_or_clone(struct send_ctx *sctx,
 		   offset < sctx->cur_inode_size) {
 		ret = send_fallocate(sctx, BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
 				     offset, len);
+	} else if (type == BTRFS_FILE_EXTENT_PREALLOC &&
+		   (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)) {
+		u32 mode = 0;
+		if (offset < sctx->cur_inode_size) {
+			ret = send_fallocate(sctx,
+					     BTRFS_SEND_PUNCH_HOLE_FALLOC_FLAGS,
+					     offset, len);
+		} else {
+			mode |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+			ret = truncate_before_falloc(sctx);
+		}
+		if (ret)
+			goto out;
+		ret = send_fallocate(sctx, mode, offset, len);
 	} else {
 		while (pos < len) {
 			l = len - pos;

@@ -4925,19 +4952,24 @@ static int process_extent(struct send_ctx *sctx,
 		ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
 				    struct btrfs_file_extent_item);
 		type = btrfs_file_extent_type(path->nodes[0], ei);
-		if (type == BTRFS_FILE_EXTENT_PREALLOC ||
-		    type == BTRFS_FILE_EXTENT_REG) {
-			/*
-			 * The send spec does not have a prealloc command yet,
-			 * so just leave a hole for prealloc'ed extents until
-			 * we have enough commands queued up to justify rev'ing
-			 * the send spec.
-			 */
-			if (type == BTRFS_FILE_EXTENT_PREALLOC) {
-				ret = 0;
-				goto out;
+		if (type == BTRFS_FILE_EXTENT_PREALLOC &&
+		    (sctx->flags & BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)) {
+			u64 len;
+			u32 flags = 0;
+
+			len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
+			if (key->offset >= sctx->cur_inode_size) {
+				flags |= BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE;
+				ret = truncate_before_falloc(sctx);
+				if (ret)
+					goto out;
 			}
-
+			ret = send_fallocate(sctx, flags, key->offset, len);
+			goto out;
+		} else if (type == BTRFS_FILE_EXTENT_PREALLOC) {
+			ret = 0;
+			goto out;
+		} else if (type == BTRFS_FILE_EXTENT_REG) {
 			/* Have a hole, just skip it. */
 			if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) {
 				ret = 0;

@@ -5123,10 +5155,13 @@ truncate_inode:
 					goto out;
 			}
 		}
-		ret = send_truncate(sctx, sctx->cur_ino, sctx->cur_inode_gen,
-				sctx->cur_inode_size);
-		if (ret < 0)
-			goto out;
+		if (!sctx->cur_inode_skip_truncate) {
+			ret = send_truncate(sctx, sctx->cur_ino,
+					    sctx->cur_inode_gen,
+					    sctx->cur_inode_size);
+			if (ret < 0)
+				goto out;
+		}
 	}
 
 	if (need_chown) {

@@ -5181,6 +5216,7 @@ static int changed_inode(struct send_ctx *sctx,
 	sctx->cur_ino = key->objectid;
 	sctx->cur_inode_new_gen = 0;
 	sctx->cur_inode_last_extent = (u64)-1;
+	sctx->cur_inode_skip_truncate = 0;
 
 	/*
 	 * Set send_progress to current inode. This will tell all get_cur_xxx

Signature · 1 line

1.9.1

Mark Fasheh

2014-04-15 17:28 UTC #9 lore raw

In reply to Filipe David Borba Manana

Quoted 10 lines · Filipe David Borba Manana

This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).

Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.

	--Mark

--
Mark Fasheh

Filipe David Manana

author

2014-04-15 17:34 UTC #10 lore raw

Quoted 9 lines · Mark Fasheh

Quoted 8 lines · Filipe David Borba Manana
This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.

Quoted 7 lines

Quoted 2 lines
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark

New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?

Quoted 3 lines

--
Mark Fasheh

Signature · 5 lines

Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

Mark Fasheh

2014-04-15 17:41 UTC #11 lore raw

Quoted 18 lines · Filipe David Manana

Quoted 9 lines · Mark Fasheh
Quoted 8 lines · Filipe David Borba Manana
This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.
Quoted 7 lines
Quoted 2 lines
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark

Quoted 4 lines

New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?

Yes it does thanks. Unfortunately though this is unacceptable behavior - kernel upgrades are not supposed to break existing userspace interfaces.

In particular what will happen here is that the user will grab a new kernel and then find out that their fancy snapshotting software won't work any more.

	--Mark

--
Mark Fasheh

Filipe David Manana

author

2014-04-15 17:57 UTC #12 lore raw

Quoted 24 lines · Mark Fasheh

Quoted 18 lines · Filipe David Manana
Quoted 9 lines · Mark Fasheh
Quoted 8 lines · Filipe David Borba Manana
This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.
Quoted 7 lines
Quoted 2 lines
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark
Quoted 4 lines
New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?

Quoted 6 lines

Yes it does thanks. Unfortunately though this is unacceptable behavior - kernel upgrades are not supposed to break existing userspace interfaces.
In particular what will happen here is that the user will grab a new kernel and then find out that their fancy snapshotting software won't work any more.

Good point. I followed this approach based on Josef's comments on a previous rfc at http://www.spinics.net/lists/linux-btrfs/msg32999.html

The only alternative I can think of right now is to use new send ioctl flags instead, so that new clients able to process the new commands will pass these flags explicitly, while old clients would continue to work without changes (and not bumping the stream version, as btrfs-receive refuses versions higher than 1 currently). This seems to be similar to what was done here:
https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/commit/?id=cb95e7bf7ba481c3d35b238b1cd671b63f54238a

thanks

Quoted 4 lines

        --Mark

--
Mark Fasheh

Signature · 5 lines

Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

Mark Fasheh

2014-04-15 18:04 UTC #13 lore raw

Quoted 13 lines · Filipe David Manana

Quoted 13 lines
Quoted 9 lines
Quoted 4 lines
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark
New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?
Yes it does thanks. Unfortunately though this is unacceptable behavior - kernel upgrades are not supposed to break existing userspace interfaces.

Quoted 6 lines

Quoted 3 lines
In particular what will happen here is that the user will grab a new kernel and then find out that their fancy snapshotting software won't work any more.
Good point. I followed this approach based on Josef's comments on a previous rfc at http://www.spinics.net/lists/linux-btrfs/msg32999.html

Apparently Josef doesn't get those sorts of bugs in his queue ;)

Quoted 7 lines

The only alternative I can think of right now is to use new send ioctl flags instead, so that new clients able to process the new commands will pass these flags explicitly, while old clients would continue to work without changes (and not bumping the stream version, as btrfs-receive refuses versions higher than 1 currently). This seems to be similar to what was done here:
https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/commit/?id=cb95e7bf7ba481c3d35b238b1cd671b63f54238a

Yeah that works for me - any client that understand the new features just sends some flag that indicates it can process them. Then we know that old clients will continue to work unaffected.

Thanks Filipe,

	--Mark

--
Mark Fasheh

Josef Bacik

2014-04-15 18:10 UTC #14 lore raw

Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,

Josef

Mark Fasheh <[email protected]> wrote:

Quoted 13 lines · Filipe David Manana

Quoted 13 lines
Quoted 9 lines
Quoted 4 lines
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark
New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?
Yes it does thanks. Unfortunately though this is unacceptable behavior - kernel upgrades are not supposed to break existing userspace interfaces.

Quoted 6 lines

Quoted 3 lines
In particular what will happen here is that the user will grab a new kernel and then find out that their fancy snapshotting software won't work any more.
Good point. I followed this approach based on Josef's comments on a
previous rfc at https://urldefense.proofpoint.com/v1/url?u=http://www.spinics.net/lists/linux-btrfs/msg32999.html&k=ZVNjlDMF0FElm4dQtryO4A%3D%3D%0A&r=cKCbChRKsMpTX8ybrSkonQ%3D%3D%0A&m=wUTU%2FIE4hdC77M9gBWyPgg1QcqkuVHjEMjOGfI79alY%3D%0A&s=4d77650ee86676b158e0f54ff93c700300f098ae6d86f36fcf61feb70fa2ac78

Apparently Josef doesn't get those sorts of bugs in his queue ;)

Quoted 7 lines

The only alternative I can think of right now is to use new send ioctl flags instead, so that new clients able to process the new commands will pass these flags explicitly, while old clients would continue to work without changes (and not bumping the stream version, as btrfs-receive refuses versions higher than 1 currently). This seems to be similar to what was done here:
https://urldefense.proofpoint.com/v1/url?u=https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/commit/?id%3Dcb95e7bf7ba481c3d35b238b1cd671b63f54238a&k=ZVNjlDMF0FElm4dQtryO4A%3D%3D%0A&r=cKCbChRKsMpTX8ybrSkonQ%3D%3D%0A&m=wUTU%2FIE4hdC77M9gBWyPgg1QcqkuVHjEMjOGfI79alY%3D%0A&s=0e82acdd655c133f52a9717216f7038a09afa1bfefaa91c4d332530a5be4c3d6

Yeah that works for me - any client that understand the new features just sends some flag that indicates it can process them. Then we know that old clients will continue to work unaffected.

Thanks Filipe,

        --Mark

--
Mark Fasheh

Filipe David Manana

author

2014-04-15 18:30 UTC #15 lore raw

Quoted 1 line · Josef Bacik

Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,

So would this flag named like "supports_v2" imply to send fallocate commands and data size computation/command? Right now I made data size optional, sent only if a new ioctl send flag is set, because for large fs trees it can take some time to compute data size.

Also, would new btrfs-progs version send that flag (support_v2) always, without any option to use old v1, or not really that useful?

thanks

Quoted 47 lines

Josef
Mark Fasheh <[email protected]> wrote:
Quoted 13 lines · Filipe David Manana
Quoted 13 lines
Quoted 9 lines
Quoted 4 lines
Are these changes compatible with software using the old stream version? We have snapshotting tools that are using send/recieve and it would be bad to change the ABI in incompatible ways underneath them.
        --Mark
New versions of btrfs-progs (send stream v2 support) will still be able to read and process v1 streams. Older btrfs-progs (v1 only) won't be able to process the new commands.
Does this answers your question Mark?
Yes it does thanks. Unfortunately though this is unacceptable behavior - kernel upgrades are not supposed to break existing userspace interfaces.
Quoted 6 lines
Quoted 3 lines
In particular what will happen here is that the user will grab a new kernel and then find out that their fancy snapshotting software won't work any more.
Good point. I followed this approach based on Josef's comments on a
previous rfc at https://urldefense.proofpoint.com/v1/url?u=http://www.spinics.net/lists/linux-btrfs/msg32999.html&k=ZVNjlDMF0FElm4dQtryO4A%3D%3D%0A&r=cKCbChRKsMpTX8ybrSkonQ%3D%3D%0A&m=wUTU%2FIE4hdC77M9gBWyPgg1QcqkuVHjEMjOGfI79alY%3D%0A&s=4d77650ee86676b158e0f54ff93c700300f098ae6d86f36fcf61feb70fa2ac78
Apparently Josef doesn't get those sorts of bugs in his queue ;)
Quoted 7 lines
The only alternative I can think of right now is to use new send ioctl flags instead, so that new clients able to process the new commands will pass these flags explicitly, while old clients would continue to work without changes (and not bumping the stream version, as btrfs-receive refuses versions higher than 1 currently). This seems to be similar to what was done here:
https://urldefense.proofpoint.com/v1/url?u=https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/commit/?id%3Dcb95e7bf7ba481c3d35b238b1cd671b63f54238a&k=ZVNjlDMF0FElm4dQtryO4A%3D%3D%0A&r=cKCbChRKsMpTX8ybrSkonQ%3D%3D%0A&m=wUTU%2FIE4hdC77M9gBWyPgg1QcqkuVHjEMjOGfI79alY%3D%0A&s=0e82acdd655c133f52a9717216f7038a09afa1bfefaa91c4d332530a5be4c3d6
Yeah that works for me - any client that understand the new features just sends some flag that indicates it can process them. Then we know that old clients will continue to work unaffected.
Thanks Filipe,
        --Mark
--
Mark Fasheh

Signature · 5 lines

Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

Mark Fasheh

2014-04-15 19:35 UTC #16 lore raw

Quoted 7 lines · Filipe David Manana

Quoted 1 line · Josef Bacik
Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,
So would this flag named like "supports_v2" imply to send fallocate commands and data size computation/command? Right now I made data size optional, sent only if a new ioctl send flag is set, because for large fs trees it can take some time to compute data size.

If it's already backwards-compatibilty you could keep the flag for data size computation as-is and simply add another flag that gets passed in for the fallocate commands (UNDERSTANDS_FALLOCATE or something lke that).

Quoted 2 lines

Also, would new btrfs-progs version send that flag (support_v2) always, without any option to use old v1, or not really that useful?

As far as btrfs-progs go you could go with many different approaches. For the library interface to this we might just let the callers determine the behavior they want. For the command processing (user runs the btrfs program directly), you could do a couple things:

- perhaps try once with the new flags, if you get an error (EINVAL right now for bad flags) try with the old ones and only then throw an error out to the caller?

- have user specify whether new flags should be tried or not (this is easiest)

I would generally avoid breaking compatibility inside of btrfs-progs too so just forcing the new flag seems the most 'breaking' option.

Thanks,

	--Mark

--
Mark Fasheh

Filipe David Manana

author

2014-04-15 19:49 UTC #17 lore raw

Quoted 4 lines · Mark Fasheh

Quoted 3 lines · Filipe David Manana
Quoted 1 line · Josef Bacik
Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,

Quoted 8 lines

Quoted 4 lines
So would this flag named like "supports_v2" imply to send fallocate commands and data size computation/command? Right now I made data size optional, sent only if a new ioctl send flag is set, because for large fs trees it can take some time to compute data size.
If it's already backwards-compatibilty you could keep the flag for data size computation as-is and simply add another flag that gets passed in for the fallocate commands (UNDERSTANDS_FALLOCATE or something lke that).

Yep, exactly what I had in mind.

Quoted 16 lines

Quoted 5 lines
Also, would new btrfs-progs version send that flag (support_v2) always, without any option to use old v1, or not really that useful?
As far as btrfs-progs go you could go with many different approaches. For the library interface to this we might just let the callers determine the behavior they want. For the command processing (user runs the btrfs program directly), you could do a couple things:
- perhaps try once with the new flags, if you get an error (EINVAL right now for bad flags) try with the old ones and only then throw an error out to the caller?

Quoted 5 lines

- have user specify whether new flags should be tried or not (this is easiest)
I would generally avoid breaking compatibility inside of btrfs-progs too so just forcing the new flag seems the most 'breaking' option.

Sounds good. Will probably try first to have the user explicitly pass a --stream-version=2 for e.g to btrfs send command, with v1 (current) being the default. I like the idea of clearing the SUPPORT_FALLOCATE/DATA_SIZE bits from the flags and retry if the ioctl returns -EINVAL too.

Thanks Mark

Quoted 6 lines

Thanks,
        --Mark
--
Mark Fasheh

Signature · 5 lines

Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

David Sterba

2014-04-18 17:18 UTC #18 lore raw

Quoted 28 lines · Filipe David Manana

Quoted 4 lines · Mark Fasheh
Quoted 3 lines · Filipe David Manana
Quoted 1 line · Josef Bacik
Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,
Quoted 8 lines
Quoted 4 lines
So would this flag named like "supports_v2" imply to send fallocate commands and data size computation/command? Right now I made data size optional, sent only if a new ioctl send flag is set, because for large fs trees it can take some time to compute data size.
If it's already backwards-compatibilty you could keep the flag for data size computation as-is and simply add another flag that gets passed in for the fallocate commands (UNDERSTANDS_FALLOCATE or something lke that).
Yep, exactly what I had in mind.
Quoted 12 lines
As far as btrfs-progs go you could go with many different approaches. For the library interface to this we might just let the callers determine the behavior they want. For the command processing (user runs the btrfs program directly), you could do a couple things:
- perhaps try once with the new flags, if you get an error (EINVAL right now for bad flags) try with the old ones and only then throw an error out to the caller?
- have user specify whether new flags should be tried or not (this is easiest)

Quoted 8 lines

Quoted 2 lines
I would generally avoid breaking compatibility inside of btrfs-progs too so just forcing the new flag seems the most 'breaking' option.
Sounds good. Will probably try first to have the user explicitly pass a --stream-version=2 for e.g to btrfs send command, with v1 (current) being the default. I like the idea of clearing the SUPPORT_FALLOCATE/DATA_SIZE bits from the flags and retry if the ioctl returns -EINVAL too.

The fallocate and progress flags are of different nature. While fallocate should be used by default with v2 (because it's more efficient and the right thing to do), the progress is optional.

So if user selects protocol version=2, he gets the falloc rightaway. Ie. no flag like VERSION_V2 or UNDERSTANDS_FALLOCATE, this should be for free under "STREAM_VERSION=2".

(I'm not sure if the falloc should be optional at all in v2.)

The progress decreases performance and I think it should be off by default.

The remaining v2-draft features are OTIME, file flags and single-file clones - all of them are not optional and would be good to add them to v2.

Filipe David Manana

author

2014-04-18 19:58 UTC #19 lore raw

Quoted 42 lines · David Sterba

Quoted 28 lines · Filipe David Manana
Quoted 4 lines · Mark Fasheh
Quoted 3 lines · Filipe David Manana
Quoted 1 line · Josef Bacik
Just make a SUPPORTS_V2 flag that userspace can pass in through the existing flags to make the kernel spit out V2 commands. We don't want to break old user space, I still have to see distro guys in real life ;). Thanks,
Quoted 8 lines
Quoted 4 lines
So would this flag named like "supports_v2" imply to send fallocate commands and data size computation/command? Right now I made data size optional, sent only if a new ioctl send flag is set, because for large fs trees it can take some time to compute data size.
If it's already backwards-compatibilty you could keep the flag for data size computation as-is and simply add another flag that gets passed in for the fallocate commands (UNDERSTANDS_FALLOCATE or something lke that).
Yep, exactly what I had in mind.
Quoted 12 lines
As far as btrfs-progs go you could go with many different approaches. For the library interface to this we might just let the callers determine the behavior they want. For the command processing (user runs the btrfs program directly), you could do a couple things:
- perhaps try once with the new flags, if you get an error (EINVAL right now for bad flags) try with the old ones and only then throw an error out to the caller?
- have user specify whether new flags should be tried or not (this is easiest)
Quoted 8 lines
Quoted 2 lines
I would generally avoid breaking compatibility inside of btrfs-progs too so just forcing the new flag seems the most 'breaking' option.
Sounds good. Will probably try first to have the user explicitly pass a --stream-version=2 for e.g to btrfs send command, with v1 (current) being the default. I like the idea of clearing the SUPPORT_FALLOCATE/DATA_SIZE bits from the flags and retry if the ioctl returns -EINVAL too.
The fallocate and progress flags are of different nature. While fallocate should be used by default with v2 (because it's more efficient and the right thing to do), the progress is optional.

Quoted 5 lines

So if user selects protocol version=2, he gets the falloc rightaway. Ie. no flag like VERSION_V2 or UNDERSTANDS_FALLOCATE, this should be for free under "STREAM_VERSION=2".
(I'm not sure if the falloc should be optional at all in v2.)

Fine for me, the only optional one in my opinion is the total data size calculation (and if requested, sets stream version to 2 automatically).

Quoted 7 lines

The progress decreases performance and I think it should be off by default.
The remaining v2-draft features are OTIME, file flags and single-file clones - all of them are not optional and would be good to add them to v2.

I'll add stubs/interfaces for them (kernel and btrfs-progs), and implement them (kernel side and btrfs-progs) later (or perhaps someone else might want to get his/her hands dirty too).

Thanks David

Signature · 5 lines

Filipe David Manana,

"Reasonable men adapt themselves to the world.
 Unreasonable men adapt the world to themselves.
 That's why all progress depends on unreasonable men."

Filipe David Borba Manana

author

2014-04-16 13:48 UTC #20 lore raw

PATCH v2 1/4 Btrfs: send, bump stream version

In reply to Filipe David Borba Manana

This increases the send stream version from version 1 to version 2, adding 2 new commands:

1) total data size - used to tell the receiver how much file data the stream will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files.

This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).

A version 2 stream is only produced if the send ioctl caller passes in one of the new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_SUPPORT_FALLOCATE), meaning old clients are unaffected.

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c            |  6 +++++-
 fs/btrfs/send.h            | 14 +++++++++++++-
 include/uapi/linux/btrfs.h | 24 +++++++++++++++++++++++-
 3 files changed, 41 insertions(+), 3 deletions(-)

fs/btrfs/send.c +5 -1

index 289e9f3..53712aa 100644
@@ -632,7 +632,11 @@ static int send_header(struct send_ctx *sctx)
 	struct btrfs_stream_header hdr;
 
 	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
-	hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);
+	if (sctx->flags & (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE |
+			   BTRFS_SEND_FLAG_SUPPORT_FALLOCATE))
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_2);
+	else
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_1);
 
 	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
 					&sctx->send_off);

fs/btrfs/send.h +13 -1

index 48d425a..367030d 100644
@@ -20,7 +20,8 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)

@@ -87,6 +88,11 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/* added in stream version 2 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)

@@ -125,10 +131,16 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/* added in stream version 2 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS,
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

include/uapi/linux/btrfs.h +23 -1

index b4d6909..6611406 100644
@@ -464,10 +464,32 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Use fallocate command to pre-allocate file extents and punch file holes,
+ * instead of write commands with data buffers filled with 0 value bytes.
+ *
+ * Added in send stream version 2.
+ */
+#define BTRFS_SEND_FLAG_SUPPORT_FALLOCATE       0x10
+
 #define BTRFS_SEND_FLAG_MASK \
 	(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-	 BTRFS_SEND_FLAG_OMIT_END_CMD)
+	 BTRFS_SEND_FLAG_OMIT_END_CMD | \
+	 BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE |	\
+	 BTRFS_SEND_FLAG_SUPPORT_FALLOCATE)
 
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-04-20 13:01 UTC #21 lore raw

PATCH v3 1/6 Btrfs: send, bump stream version

This increases the send stream version from version 1 to version 2, adding new commands:

1) total data size - used to tell the receiver how much file data the stream will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

A version 2 stream is only produced if the send ioctl caller passes in one of the new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_STREAM_V2), meaning old clients are unaffected.

Signed-off-by Filipe David Borba Manana <[email protected]>

---

 fs/btrfs/send.c            |  7 ++++++-
 fs/btrfs/send.h            | 21 ++++++++++++++++++++-
 include/uapi/linux/btrfs.h | 21 ++++++++++++++++++++-
 3 files changed, 46 insertions(+), 3 deletions(-)

fs/btrfs/send.c +6 -1

index 289e9f3..7b4b0c3 100644
@@ -632,7 +632,10 @@ static int send_header(struct send_ctx *sctx)
 	struct btrfs_stream_header hdr;
 
 	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
-	hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);
+	if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_2);
+	else
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_1);
 
 	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
 					&sctx->send_off);

@@ -5554,6 +5557,8 @@ long btrfs_ioctl_send(struct file *mnt_file, void __user *arg_)
 	INIT_LIST_HEAD(&sctx->name_cache_list);
 
 	sctx->flags = arg->flags;
+	if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE)
+		sctx->flags |= BTRFS_SEND_FLAG_STREAM_V2;
 
 	sctx->send_filp = fget(arg->send_fd);
 	if (!sctx->send_filp) {

fs/btrfs/send.h +20 -1

index 48d425a..96f583c 100644
@@ -20,7 +20,8 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)

@@ -87,6 +88,15 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/*
+	 * The following commands were added in stream version 2.
+	 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+	BTRFS_SEND_C_INODE_SET_FLAGS,
+	BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)

@@ -125,10 +135,19 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/*
+	 * The following attributes were added in stream version 2.
+	 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
+	BTRFS_SEND_A_INODE_FLAGS,     /* 32 bits */
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

include/uapi/linux/btrfs.h +20 -1

index b4d6909..8ab2761 100644
@@ -464,10 +464,29 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ *
+ * Added in send stream version 2, and implies producing a version 2 stream.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2              0x10
+
 #define BTRFS_SEND_FLAG_MASK \
 	(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-	 BTRFS_SEND_FLAG_OMIT_END_CMD)
+	 BTRFS_SEND_FLAG_OMIT_END_CMD | \
+	 BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | \
+	 BTRFS_SEND_FLAG_STREAM_V2)
 
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */

Signature · 1 line

1.9.1

Filipe David Borba Manana

author

2014-06-23 11:00 UTC #22 lore raw

PATCH v5 1/6 Btrfs: send, bump stream version

In reply to Filipe David Borba Manana

This increases the send stream version from version 1 to version 2, adding new commands:

1) total data size - used to tell the receiver how much file data the stream will add or update;

2) fallocate - used to pre-allocate space for files and to punch holes in files;

3) inode set flags;

4) set inode otime.

This is preparation work for subsequent changes that implement the new features.

A version 2 stream is only produced if the send ioctl caller passes in one of the new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_STREAM_V2), meaning old clients are unaffected.

Signed-off-by Filipe David Borba Manana <[email protected]>

---

V2: A v2 stream is now only produced if the send ioctl caller passes in one of
the new flags (BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | BTRFS_SEND_FLAG_SUPPORT_FALLOCATE) to avoid breaking old clients.
V3: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE and added BTRFS_SEND_FLAG_STREAM_V2,
added commands for inode set flags and otime.
V4: There's no v4, bumped directly to v5 to make all patches in the series have the same
version.
V5: Rebased against latest chris/integration branch.

 fs/btrfs/send.c            |  7 ++++++-
 fs/btrfs/send.h            | 21 ++++++++++++++++++++-
 include/uapi/linux/btrfs.h | 21 ++++++++++++++++++++-
 3 files changed, 46 insertions(+), 3 deletions(-)

fs/btrfs/send.c +6 -1

index 6528aa6..d7ef14b 100644
@@ -637,7 +637,10 @@ static int send_header(struct send_ctx *sctx)
 	struct btrfs_stream_header hdr;
 
 	strcpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC);
-	hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION);
+	if (sctx->flags & BTRFS_SEND_FLAG_STREAM_V2)
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_2);
+	else
+		hdr.version = cpu_to_le32(BTRFS_SEND_STREAM_VERSION_1);
 
 	return write_buf(sctx->send_filp, &hdr, sizeof(hdr),
 					&sctx->send_off);

@@ -5572,6 +5575,8 @@ long btrfs_ioctl_send(struct file *mnt_file, void __user *arg_)
 	INIT_LIST_HEAD(&sctx->name_cache_list);
 
 	sctx->flags = arg->flags;
+	if (sctx->flags & BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE)
+		sctx->flags |= BTRFS_SEND_FLAG_STREAM_V2;
 
 	sctx->send_filp = fget(arg->send_fd);
 	if (!sctx->send_filp) {

fs/btrfs/send.h +20 -1

index 48d425a..96f583c 100644
@@ -20,7 +20,8 @@
 #include "ctree.h"
 
 #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream"
-#define BTRFS_SEND_STREAM_VERSION 1
+#define BTRFS_SEND_STREAM_VERSION_1 1
+#define BTRFS_SEND_STREAM_VERSION_2 2
 
 #define BTRFS_SEND_BUF_SIZE (1024 * 64)
 #define BTRFS_SEND_READ_SIZE (1024 * 48)

@@ -87,6 +88,15 @@ enum btrfs_send_cmd {
 
 	BTRFS_SEND_C_END,
 	BTRFS_SEND_C_UPDATE_EXTENT,
+
+	/*
+	 * The following commands were added in stream version 2.
+	 */
+	BTRFS_SEND_C_TOTAL_DATA_SIZE,
+	BTRFS_SEND_C_FALLOCATE,
+	BTRFS_SEND_C_INODE_SET_FLAGS,
+	BTRFS_SEND_C_UTIMES2, /* Same as UTIMES, but it includes OTIME too. */
+
 	__BTRFS_SEND_C_MAX,
 };
 #define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)

@@ -125,10 +135,19 @@ enum {
 	BTRFS_SEND_A_CLONE_OFFSET,
 	BTRFS_SEND_A_CLONE_LEN,
 
+	/*
+	 * The following attributes were added in stream version 2.
+	 */
+	BTRFS_SEND_A_FALLOCATE_FLAGS, /* 32 bits */
+	BTRFS_SEND_A_INODE_FLAGS,     /* 32 bits */
+
 	__BTRFS_SEND_A_MAX,
 };
 #define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1)
 
+#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE   (1 << 0)
+#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE  (1 << 1)
+
 #ifdef __KERNEL__
 long btrfs_ioctl_send(struct file *mnt_file, void __user *arg);
 #endif

include/uapi/linux/btrfs.h +20 -1

index 6f9c38c..62440d8 100644
@@ -477,10 +477,29 @@ struct btrfs_ioctl_received_subvol_args {
  */
 #define BTRFS_SEND_FLAG_OMIT_END_CMD		0x4
 
+/*
+ * Calculate the amount (in bytes) of new file data between the send and
+ * parent snapshots, or in case of a full send, the total amount of file data
+ * we will send.
+ * This corresponds to the sum of the data lengths of each write, clone and
+ * fallocate commands that are sent through the send stream. The receiving end
+ * can use this information to compute progress.
+ *
+ * Added in send stream version 2, and implies producing a version 2 stream.
+ */
+#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE	0x8
+
+/*
+ * Used by a client to request a version 2 of the send stream.
+ */
+#define BTRFS_SEND_FLAG_STREAM_V2              0x10
+
 #define BTRFS_SEND_FLAG_MASK \
 	(BTRFS_SEND_FLAG_NO_FILE_DATA | \
 	 BTRFS_SEND_FLAG_OMIT_STREAM_HEADER | \
-	 BTRFS_SEND_FLAG_OMIT_END_CMD)
+	 BTRFS_SEND_FLAG_OMIT_END_CMD | \
+	 BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE | \
+	 BTRFS_SEND_FLAG_STREAM_V2)
 
 struct btrfs_ioctl_send_args {
 	__s64 send_fd;			/* in */

Signature · 1 line

1.9.1