Btrfs-progs: send, bump stream version
Subsystems (2)
PATCH 1/4 Btrfs-progs: send, bump stream version
This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
--- ioctl.h | 8 ++++++++ send.h | 13 ++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-)
ioctl.h +8 -0
@@ -392,6 +392,14 @@ struct btrfs_ioctl_received_subvol_args {*/ #define BTRFS_SEND_FLAG_OMIT_END_CMD 0x4 +/* +* The sum of all length fields the receiver will get in write, clone and +* fallocate commands. +* This can be used by the receiver to compute progress, at the expense of some +* initial metadata scan performed by the sender (kernel). +*/ +#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE 0x8 +struct btrfs_ioctl_send_args {__s64 send_fd;/* in */ __u64 clone_sources_count;/* in */
send.h +12 -1
@@ -24,7 +24,7 @@ extern "C" {#endif #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" -#define BTRFS_SEND_STREAM_VERSION 1 +#define BTRFS_SEND_STREAM_VERSION 2#define BTRFS_SEND_BUF_SIZE (1024 * 64)#define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +91,11 @@ enum btrfs_send_cmd {BTRFS_SEND_C_END ,BTRFS_SEND_C_UPDATE_EXTENT , + +/* added in stream version 2 */ +BTRFS_SEND_C_TOTAL_DATA_SIZE , +BTRFS_SEND_C_FALLOCATE , + __BTRFS_SEND_C_MAX, };#define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +134,16 @@ enum {BTRFS_SEND_A_CLONE_OFFSET ,BTRFS_SEND_A_CLONE_LEN , +/* added in stream version 2 */ +BTRFS_SEND_A_FALLOCATE_FLAGS , + __BTRFS_SEND_A_MAX, };#define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1) +#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE (1 << 0) +#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE (1 << 1) +#ifdef __KERNEL__long btrfs_ioctl_send (struct file *mnt_file,void __user *arg);#endif
Signature · 1 line
1.9.1
PATCH 2/4 Btrfs-progs: send, implement total data size callback and progress report
This is a followup to the kernel patch titled:
Btrfs: send, implement total data size command to allow for progress estimation
This makes the btrfs send and receive commands aware of the new send flag, named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data that is new between the parent and send snapshots/roots. As this command immediately follows the commands to start a snapshot/subvolume, it can be used to report and compute progress, by keeping a counter that is incremented with the data length of each write, clone and fallocate command that is received from the stream.
Example:
$ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/sAt the moment progress is only reported by btrfs-receive, but it is possible and simple to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send output to btrfs-receive (directly to a file).
--- Documentation/btrfs-send.txt | 3 ++ cmds-receive.c | 91 ++++++++++++++++++++++++++++++++++++++++++++ cmds-send.c | 14 ++++++- send-stream.c | 4 ++ send-stream.h | 1 + 5 files changed, 111 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed). -f <outfile>:: Output is normally written to stdout. To write to a file, use this option. An alternative would be to use pipes. +-o:: +Obtain the total data size for each subvolume or snapshot to send. This demands additional +processing (mostly IO bound) but is useful for the receive command to report progress. EXIT STATUS -----------
cmds-receive.c +91 -0
139 lines not on this page · raw message
cmds-send.c +12 -2
@@ -45,6 +45,7 @@#include "send-utils.h" static int g_verbose =0 ; +static int get_total_data_size =0 ;struct btrfs_send {int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_STREAM_HEADER ;if (!is_last_subvol) io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ; +if (get_total_data_size) + io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt (argc, argv,"vec:f:i:p:" )) !=-1 ) { +while ((c =getopt (argc, argv,"veoc:f:i:p:" )) !=-1 ) {switch (c) {case 'v' : g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv) "ERROR: -i was removed, use -c instead\n"); ret =1 ; goto out; +case 'o' : + get_total_data_size =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -673,7 +679,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {"-f <outfile> Output is normally written to stdout. To write to" ," a file, use this option. An alternative would be to" ," use pipes." , +"-o Obtain the total data size for each subvolume or " , +" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress." , NULL };
send-stream.c +4 -0
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->update_extent (path, offset, tmp, s->user);break ; +case BTRFS_SEND_C_TOTAL_DATA_SIZE : +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); + ret = s->ops->total_data_size (tmp, s->user); +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +1 -0
@@ -54,6 +54,7 @@ struct btrfs_send_ops { struct timespec *mt, struct timespec *ct, void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user); +int (*total_data_size)(u64 size,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
PATCH v2 2/4 Btrfs-progs: send, implement total data size callback and progress report
This is a followup to the kernel patch titled:
Btrfs: send, implement total data size command to allow for progress estimation
This makes the btrfs send and receive commands aware of the new send flag, named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data that is new between the parent and send snapshots/roots. As this command immediately follows the commands to start a snapshot/subvolume, it can be used to report and compute progress, by keeping a counter that is incremented with the data length of each write, clone and fallocate command that is received from the stream.
Example:
$ btrfs send -o /mnt/sdd/snap_base | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s $ btrfs send -o -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/sAt the moment progress is only reported by btrfs-receive, but it is possible and simple to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send output to btrfs-receive (directly to a file).
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
Documentation/btrfs-send.txt | 3 ++ cmds-receive.c | 91 ++++++++++++++++++++++++++++++++++++++++++++ cmds-send.c | 14 ++++++- send-stream.c | 4 ++ send-stream.h | 1 + 5 files changed, 111 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed). -f <outfile>:: Output is normally written to stdout. To write to a file, use this option. An alternative would be to use pipes. +-o:: +Obtain the total data size for each subvolume or snapshot to send. This demands additional +processing (mostly IO bound) but is useful for the receive command to report progress. EXIT STATUS -----------
cmds-receive.c +91 -0
139 lines not on this page · raw message
cmds-send.c +12 -2
@@ -45,6 +45,7 @@#include "send-utils.h" static int g_verbose =0 ; +static int g_total_data_size =0 ;struct btrfs_send {int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_STREAM_HEADER ;if (!is_last_subvol) io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ; +if (g_total_data_size) + io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt (argc, argv,"vec:f:i:p:" )) !=-1 ) { +while ((c =getopt (argc, argv,"veoc:f:i:p:" )) !=-1 ) {switch (c) {case 'v' : g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv) "ERROR: -i was removed, use -c instead\n"); ret =1 ; goto out; +case 'o' : + g_total_data_size =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -673,7 +679,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {"-f <outfile> Output is normally written to stdout. To write to" ," a file, use this option. An alternative would be to" ," use pipes." , +"-o Obtain the total data size for each subvolume or " , +" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress." , NULL };
send-stream.c +4 -0
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->update_extent (path, offset, tmp, s->user);break ; +case BTRFS_SEND_C_TOTAL_DATA_SIZE : +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); + ret = s->ops->total_data_size (tmp, s->user); +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +1 -0
@@ -54,6 +54,7 @@ struct btrfs_send_ops { struct timespec *mt, struct timespec *ct, void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user); +int (*total_data_size)(u64 size,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
Re: [PATCH 2/4 v2] Btrfs-progs: send, implement total data size callback and progress report
Quoted 4 lines · Filipe David Borba Manana
V2: Added new send ioctl flag
BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATEorBTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
Good.
Quoted 14 lines
cmds-receive.c +6
-0@@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, goto out; r->cur_subvol =calloc (1 ,sizeof (*r->cur_subvol)); + r->total_data_size =0 ; + r->bytes_received =0 ; + r->progress =0.0 ; + r->last_progress_update =0 ; + r->bytes_received_last_update =0 ; + r->target ="Subvolume" ;if (strlen (r->dest_dir_path) ==0 ) r->cur_subvol->path =strdup (path);
cmds-receive.c static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, · line 228
Quoted 9 lines
cmds-receive.c +6
-0@@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, goto out; r->cur_subvol =calloc (1 ,sizeof (*r->cur_subvol)); + r->total_data_size =0 ; + r->bytes_received =0 ; + r->progress =0.0 ; + r->last_progress_update =0 ; + r->bytes_received_last_update =0 ; + r->target ="Snapshot" ;
Nontrivial amount of duplicate code, a helper would be better.
Quoted 10 lines
cmds-send.c +1
-1@@ -673,7 +679,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
cmds-send.c const char * const cmd_send_usage[] = { · line 706
Quoted 4 lines
cmds-send.c +1
-0@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {"-f <outfile> Output is normally written to stdout. To write to" ," a file, use this option. An alternative would be to" ," use pipes." , + "-o Obtain the total data size for each subvolume or ",
UI:
-o is sometimes used as an option for 'output', which makes sense in context of send, but is already done via -f. I'm not sure if it's a good choice. 'p' is already occupied.
As an alternative: how about -s ? mnemonic for 'size'.
Quoted 5 lines
cmds-send.c +3
-0+" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress." , NULL };
Re: [PATCH 2/4 v2] Btrfs-progs: send, implement total data size callback and progress report
Quoted 24 lines · David Sterba
Quoted 4 lines · Filipe David Borba Manana
V2: Added new send ioctl flag
BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATEorBTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).Good.
Quoted 16 lines
cmds-receive.c +6
-0@@ -156,6 +165,12 @@ static int process_subvol(const char *path, const u8 *uuid, u64 ctransid, goto out; r->cur_subvol =calloc (1 ,sizeof (*r->cur_subvol)); + r->total_data_size =0 ; + r->bytes_received =0 ; + r->progress =0.0 ; + r->last_progress_update =0 ; + r->bytes_received_last_update =0 ; + r->target ="Subvolume" ;if (strlen (r->dest_dir_path) ==0 ) r->cur_subvol->path =strdup (path);@@ -205,6 +220,12 @@ static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, goto out;
Quoted 9 lines
cmds-receive.c static int process_snapshot(const char *path, const u8 *uuid, u64 ctransid, · line 228
Quoted 7 lines
cmds-receive.c +6
-0r->cur_subvol =calloc (1 ,sizeof (*r->cur_subvol)); + r->total_data_size =0 ; + r->bytes_received =0 ; + r->progress =0.0 ; + r->last_progress_update =0 ; + r->bytes_received_last_update =0 ; + r->target ="Snapshot" ;Nontrivial amount of duplicate code, a helper would be better.
Agree about the duplication, but can't agree with being non-trivial:
resetting to 0 a few counters/percentage/timestamp :)
Quoted 18 lines
Quoted 11 lines
cmds-send.c +1
-1@@ -673,7 +679,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,cmds-send.c const char * const cmd_send_usage[] = { · line 706
Quoted 4 lines
cmds-send.c +1
-0@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {"-f <outfile> Output is normally written to stdout. To write to" ," a file, use this option. An alternative would be to" ," use pipes." , + "-o Obtain the total data size for each subvolume or ",UI:
Quoted 5 lines
-o is sometimes used as an option for 'output', which makes sense in context of send, but is already done via -f. I'm not sure if it's a good choice. 'p' is already occupied.
As an alternative: how about -s ? mnemonic for 'size'.
Fine for me. Don't have a strong opinion about the letter.
thanks David
Quoted 6 lines
Quoted 6 lines
cmds-send.c +3
-0+" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress." , NULL };
Signature · 5 lines
Filipe David Manana, "Reasonable men adapt themselves to the world. Unreasonable men adapt the world to themselves. That's why all progress depends on unreasonable men."
PATCH v3 2/4 Btrfs-progs: send, implement total data size callback and progress report
In reply to Filipe David Borba MananaThis is a followup to the kernel patch titled:
Btrfs: send, implement total data size command to allow for progress estimation
This makes the btrfs send and receive commands aware of the new send flag, named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data that is new between the parent and send snapshots/roots. As this command immediately follows the commands to start a snapshot/subvolume, it can be used to report and compute progress, by keeping a counter that is incremented with the data length of each write, clone and fallocate command that is received from the stream.
Example:
$ btrfs send -s /mnt/sdd/snap_base | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s $ btrfs send -s -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/sAt the moment progress is only reported by btrfs-receive, but it is possible and simple to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send output to btrfs-receive (directly to a file).
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Renamed option -o to -s, removed some duplicated code (progress reset).
Documentation/btrfs-send.txt | 3 ++ cmds-receive.c | 91 ++++++++++++++++++++++++++++++++++++++++++++ cmds-send.c | 14 ++++++- send-stream.c | 4 ++ send-stream.h | 1 + 5 files changed, 111 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed). -f <outfile>:: Output is normally written to stdout. To write to a file, use this option. An alternative would be to use pipes. +-s:: +Obtain the total data size for each subvolume or snapshot to send. This demands additional +processing (mostly IO bound) but is useful for the receive command to report progress. EXIT STATUS -----------
cmds-receive.c +91 -0
145 lines not on this page · raw message
cmds-send.c +12 -2
@@ -45,6 +45,7 @@#include "send-utils.h" static int g_verbose =0 ; +static int g_total_data_size =0 ;struct btrfs_send {int send_fd;
@@ -281,6 +282,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_STREAM_HEADER ;if (!is_last_subvol) io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ; +if (g_total_data_size) + io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -424,7 +427,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt (argc, argv,"vec:f:i:p:" )) !=-1 ) { +while ((c =getopt (argc, argv,"vesc:f:i:p:" )) !=-1 ) {switch (c) {case 'v' : g_verbose++;
@@ -511,6 +514,9 @@ int cmd_send(int argc, char **argv) "ERROR: -i was removed, use -c instead\n"); ret =1 ; goto out; +case 's' : + g_total_data_size =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -673,7 +679,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-ves] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -697,5 +703,9 @@ const char * const cmd_send_usage[] = {"-f <outfile> Output is normally written to stdout. To write to" ," a file, use this option. An alternative would be to" ," use pipes." , +"-s Obtain the total data size for each subvolume or " , +" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress." , NULL };
send-stream.c +4 -0
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->update_extent (path, offset, tmp, s->user);break ; +case BTRFS_SEND_C_TOTAL_DATA_SIZE : +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); + ret = s->ops->total_data_size (tmp, s->user); +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +1 -0
@@ -54,6 +54,7 @@ struct btrfs_send_ops { struct timespec *mt, struct timespec *ct, void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user); +int (*total_data_size)(u64 size,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
PATCH v4 2/4 Btrfs-progs: send, implement total data size callback and progress report
In reply to Filipe David Borba MananaThis is a followup to the kernel patch titled:
Btrfs: send, implement total data size command to allow for progress estimation
This makes the btrfs send and receive commands aware of the new send flag, named BTRFS_SEND_C_TOTAL_DATA_SIZE, which tells us the amount of file data that is new between the parent and send snapshots/roots. As this command immediately follows the commands to start a snapshot/subvolume, it can be used to report and compute progress, by keeping a counter that is incremented with the data length of each write, clone and fallocate command that is received from the stream.
Example:
$ btrfs send -s --stream-version 2 /mnt/sdd/snap_base | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_base
At subvol snap_base
About to receive 9212392667 bytes
Subvolume /mnt/sdc//snap_base, 4059722426 / 9212392667 bytes received, 44.07%, 40.32MB/s $ btrfs send -s --stream-version 2 -p /mnt/sdd/snap_base /mnt/sdd/snap_incr | btrfs receive /mnt/sdc
At subvol /mnt/sdd/snap_incr
At subvol snap_incr
About to receive 9571342213 bytes
Subvolume /mnt/sdc//snap_incr, 6557345221 / 9571342213 bytes received, 68.51%, 51.04MB/sAt the moment progress is only reported by btrfs-receive, but it is possible and simple to do it for btrfs-send too, so that we can get progress report when not piping btrfs-send output to btrfs-receive (directly to a file).
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Renamed option -o to -s, removed some duplicated code (progress reset).
V4: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and --stream-version=<version_number> respectively. Added commands for inode set flags and otime too.
Documentation/btrfs-send.txt | 4 ++ cmds-receive.c | 91 ++++++++++++++++++++++++++++++++++++++++++++ cmds-send.c | 23 ++++++++++- send-stream.c | 4 ++ send-stream.h | 1 + 5 files changed, 121 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +4 -0
@@ -43,6 +43,10 @@ An alternative would be to use pipes. --stream-version <version>:: Ask the kernel to produce a specific send stream version. More recent stream versions provide new features and better performance. Default value is 1. +-s:: +Obtain the total data size for each subvolume or snapshot to send. This demands additional +processing (mostly IO bound) but is useful for the receive command to report progress. This +option requires send stream version 2 or higher. EXIT STATUS -----------
cmds-receive.c +91 -0
145 lines not on this page · raw message
cmds-send.c +21 -2
@@ -47,6 +47,7 @@static int g_verbose =0 ;static int g_stream_version =BTRFS_SEND_STREAM_VERSION_1 ; +static int g_total_data_size =0 ;struct btrfs_send {int send_fd;
@@ -285,6 +286,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ;if (g_stream_version ==BTRFS_SEND_STREAM_VERSION_2 ) io_send.flags |=BTRFS_SEND_FLAG_STREAM_V2 ; +if (g_total_data_size) + io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -433,7 +436,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt_long (argc, argv,"vec:f:i:p:" , +while ((c =getopt_long (argc, argv,"vesc:f:i:p:" , long_options,NULL )) !=-1 ) {switch (c) {case 'v' :
@@ -539,6 +542,9 @@ int cmd_send(int argc, char **argv) goto out; }break ; +case 's' : + g_total_data_size =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -550,6 +556,14 @@ int cmd_send(int argc, char **argv)if (optind == argc)usage (cmd_send_usage); +if (g_total_data_size && + g_stream_version <BTRFS_SEND_STREAM_VERSION_2 ) { +fprintf (stderr, +"ERROR: option total data size (-s) requires use of the send stream version 2 or higher\n" ); + ret =1 ; + goto out; + } +if (outname !=NULL ) { send.dump_fd =creat (outname,0600 );if (send.dump_fd ==-1 ) {
@@ -701,7 +715,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ve] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-ves] [--stream-version <version>] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -728,5 +742,10 @@ const char * const cmd_send_usage[] = {"--stream-version <version> Ask the kernel to produce a specific send stream" ," version. More recent stream versions provide new" ," features and better performance. Default value is 1." , +"-s Obtain the total data size for each subvolume or " , +" snapshot to send. This demands additional processing" , +" (mostly IO bound) but is useful for the receive " , +" command to report progress. This option requires use" , +" of the send stream version 2 or higher." ,NULL };
send-stream.c +4 -0
@@ -421,6 +421,10 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->update_extent (path, offset, tmp, s->user);break ; +case BTRFS_SEND_C_TOTAL_DATA_SIZE : +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); + ret = s->ops->total_data_size (tmp, s->user); +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +1 -0
@@ -54,6 +54,7 @@ struct btrfs_send_ops { struct timespec *mt, struct timespec *ct, void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user); +int (*total_data_size)(u64 size,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 6 lines
1.9.1 -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo
PATCH 3/4 Btrfs-progs: send, implement fallocate command callback
In reply to Filipe David Borba MananaThe fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
--- cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++ send-stream.c | 13 +++++++++++++ send-stream.h | 2 ++ 3 files changed, 53 insertions(+)
cmds-receive.c +38 -0
@@ -41,6 +41,7 @@#include <sys/types.h> #include <sys/xattr.h> #include <uuid/uuid.h> +#include <linux/falloc.h> #include "ctree.h" #include "ioctl.h"
@@ -887,6 +888,42 @@ out:return ret; } +static int process_fallocate (const char *path,u32 flags,u64 offset, +u64 len,void *user) +{ +struct btrfs_receive *r = user; +char *full_path =path_cat (r->full_subvol_path, path); +int mode =0 ; +int ret; + +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE ) + mode |=FALLOC_FL_KEEP_SIZE ; +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE ) + mode |=FALLOC_FL_PUNCH_HOLE ; + +if (g_verbose >=2 ) +fprintf (stderr, +"fallocate %s - flags %u, offset %llu, len %llu\n" , + path, flags, offset, len); + + ret =open_inode_for_write (r, full_path); +if (ret <0 ) + goto out; + + ret =fallocate (r->write_fd, mode, offset, len); +if (ret) { + ret = -errno; +fprintf (stderr, +"ERROR: fallocate against %s failed. %s\n" , + path,strerror (-ret)); + goto out; + } +update_progress (r, len); + +out: +free (full_path); +return ret; +}static struct btrfs_send_ops send_ops = { .subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = { .chown = process_chown, .utimes = process_utimes, .total_data_size = process_total_data_size, + .fallocate = process_fallocate, };static int do_receive (struct btrfs_receive *r,const char *tomnt,int r_fd)
send-stream.c +13 -0
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->total_data_size (tmp, s->user);break ; +case BTRFS_SEND_C_FALLOCATE : + { +u32 flags; +u64 len; + +TLV_GET_STRING (s,BTRFS_SEND_A_PATH , &path); +TLV_GET_U32 (s,BTRFS_SEND_A_FALLOCATE_FLAGS , &flags); +TLV_GET_U64 (s,BTRFS_SEND_A_FILE_OFFSET , &offset); +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &len); + ret = s->ops->fallocate (path, flags, offset, len, + s->user); + } +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +2 -0
@@ -55,6 +55,8 @@ struct btrfs_send_ops { void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user);int (*total_data_size)(u64 size,void *user); +int (*fallocate)(const char *path,u32 flags,u64 offset, +u64 len,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
PATCH v2 3/4 Btrfs-progs: send, implement fallocate command callback
The fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
---
V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
asks for it (-a command line option), which will make the kernel generate a version 2 send stream, so that old clients aren't affected.
Documentation/btrfs-send.txt | 3 +++ cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++ cmds-send.c | 12 ++++++++++-- send-stream.c | 13 +++++++++++++ send-stream.h | 2 ++ 5 files changed, 66 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -43,6 +43,9 @@ An alternative would be to use pipes. -o:: Obtain the total data size for each subvolume or snapshot to send. This demands additional processing (mostly IO bound) but is useful for the receive command to report progress. +-a:: +Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes +to files. EXIT STATUS -----------
cmds-receive.c +38 -0
@@ -41,6 +41,7 @@#include <sys/types.h> #include <sys/xattr.h> #include <uuid/uuid.h> +#include <linux/falloc.h> #include "ctree.h" #include "ioctl.h"
@@ -887,6 +888,42 @@ out:return ret; } +static int process_fallocate (const char *path,u32 flags,u64 offset, +u64 len,void *user) +{ +struct btrfs_receive *r = user; +char *full_path =path_cat (r->full_subvol_path, path); +int mode =0 ; +int ret; + +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE ) + mode |=FALLOC_FL_KEEP_SIZE ; +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE ) + mode |=FALLOC_FL_PUNCH_HOLE ; + +if (g_verbose >=2 ) +fprintf (stderr, +"fallocate %s - flags %u, offset %llu, len %llu\n" , + path, flags, offset, len); + + ret =open_inode_for_write (r, full_path); +if (ret <0 ) + goto out; + + ret =fallocate (r->write_fd, mode, offset, len); +if (ret) { + ret = -errno; +fprintf (stderr, +"ERROR: fallocate against %s failed. %s\n" , + path,strerror (-ret)); + goto out; + } +update_progress (r, len); + +out: +free (full_path); +return ret; +}static struct btrfs_send_ops send_ops = { .subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = { .chown = process_chown, .utimes = process_utimes, .total_data_size = process_total_data_size, + .fallocate = process_fallocate, };static int do_receive (struct btrfs_receive *r,const char *tomnt,int r_fd)
cmds-send.c +10 -2
@@ -46,6 +46,7 @@static int g_verbose =0 ;static int g_total_data_size =0 ; +static int g_fallocate =0 ;struct btrfs_send {int send_fd;
@@ -284,6 +285,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ;if (g_total_data_size) io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; +if (g_fallocate) + io_send.flags |=BTRFS_SEND_FLAG_SUPPORT_FALLOCATE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -427,7 +430,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt (argc, argv,"veoc:f:i:p:" )) !=-1 ) { +while ((c =getopt (argc, argv,"veoac:f:i:p:" )) !=-1 ) {switch (c) {case 'v' : g_verbose++;
@@ -517,6 +520,9 @@ int cmd_send(int argc, char **argv)case 'o' : g_total_data_size =1 ;break ; +case 'a' : + g_fallocate =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -679,7 +685,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-veo] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-veoa] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -707,5 +713,7 @@ const char * const cmd_send_usage[] = {" snapshot to send. This demands additional processing" ," (mostly IO bound) but is useful for the receive " ," command to report progress." , +"-a Use fallocate to pre-allocate file extents and to" , +" punch file holes, instead of writing zeroes to files." , NULL };
send-stream.c +13 -0
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->total_data_size (tmp, s->user);break ; +case BTRFS_SEND_C_FALLOCATE : + { +u32 flags; +u64 len; + +TLV_GET_STRING (s,BTRFS_SEND_A_PATH , &path); +TLV_GET_U32 (s,BTRFS_SEND_A_FALLOCATE_FLAGS , &flags); +TLV_GET_U64 (s,BTRFS_SEND_A_FILE_OFFSET , &offset); +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &len); + ret = s->ops->fallocate (path, flags, offset, len, + s->user); + } +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +2 -0
@@ -55,6 +55,8 @@ struct btrfs_send_ops { void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user);int (*total_data_size)(u64 size,void *user); +int (*fallocate)(const char *path,u32 flags,u64 offset, +u64 len,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
PATCH v3 3/4 Btrfs-progs: send, implement fallocate command callback
In reply to Filipe David Borba MananaThe fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
---
V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
asks for it (-a command line option), which will make the kernel generate a version 2 send stream, so that old clients aren't affected.
V3: Rebased on new patchset (new version of patch 2/4).
Documentation/btrfs-send.txt | 3 +++ cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++ cmds-send.c | 12 ++++++++++-- send-stream.c | 13 +++++++++++++ send-stream.h | 2 ++ 5 files changed, 66 insertions(+), 2 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -43,6 +43,9 @@ An alternative would be to use pipes. -s:: Obtain the total data size for each subvolume or snapshot to send. This demands additional processing (mostly IO bound) but is useful for the receive command to report progress. +-a:: +Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes +to files. EXIT STATUS -----------
cmds-receive.c +38 -0
@@ -41,6 +41,7 @@#include <sys/types.h> #include <sys/xattr.h> #include <uuid/uuid.h> +#include <linux/falloc.h> #include "ctree.h" #include "ioctl.h"
@@ -887,6 +888,42 @@ out:return ret; } +static int process_fallocate (const char *path,u32 flags,u64 offset, +u64 len,void *user) +{ +struct btrfs_receive *r = user; +char *full_path =path_cat (r->full_subvol_path, path); +int mode =0 ; +int ret; + +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE ) + mode |=FALLOC_FL_KEEP_SIZE ; +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE ) + mode |=FALLOC_FL_PUNCH_HOLE ; + +if (g_verbose >=2 ) +fprintf (stderr, +"fallocate %s - flags %u, offset %llu, len %llu\n" , + path, flags, offset, len); + + ret =open_inode_for_write (r, full_path); +if (ret <0 ) + goto out; + + ret =fallocate (r->write_fd, mode, offset, len); +if (ret) { + ret = -errno; +fprintf (stderr, +"ERROR: fallocate against %s failed. %s\n" , + path,strerror (-ret)); + goto out; + } +update_progress (r, len); + +out: +free (full_path); +return ret; +}static struct btrfs_send_ops send_ops = { .subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = { .chown = process_chown, .utimes = process_utimes, .total_data_size = process_total_data_size, + .fallocate = process_fallocate, };static int do_receive (struct btrfs_receive *r,const char *tomnt,int r_fd)
cmds-send.c +10 -2
@@ -46,6 +46,7 @@static int g_verbose =0 ;static int g_total_data_size =0 ; +static int g_fallocate =0 ;struct btrfs_send {int send_fd;
@@ -284,6 +285,8 @@ static int do_send(struct btrfs_send *send, u64 parent_root_id, io_send.flags |=BTRFS_SEND_FLAG_OMIT_END_CMD ;if (g_total_data_size) io_send.flags |=BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE ; +if (g_fallocate) + io_send.flags |=BTRFS_SEND_FLAG_SUPPORT_FALLOCATE ; ret =ioctl (subvol_fd,BTRFS_IOC_SEND , &io_send);if (ret) { ret = -errno;
@@ -427,7 +430,7 @@ int cmd_send(int argc, char **argv)memset (&send,0 ,sizeof (send)); send.dump_fd =fileno (stdout); -while ((c =getopt (argc, argv,"vesc:f:i:p:" )) !=-1 ) { +while ((c =getopt (argc, argv,"vesac:f:i:p:" )) !=-1 ) {switch (c) {case 'v' : g_verbose++;
@@ -517,6 +520,9 @@ int cmd_send(int argc, char **argv)case 's' : g_total_data_size =1 ;break ; +case 'a' : + g_fallocate =1 ; +break ;case '?' :default :fprintf (stderr,"ERROR: send args invalid.\n" );
@@ -679,7 +685,7 @@ out: }const char *const cmd_send_usage[] = { -"btrfs send [-ves] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" , +"btrfs send [-vesa] [-p <parent>] [-c <clone-src>] [-f <outfile>] <subvol> [<subvol>...]" ,"Send the subvolume(s) to stdout." ,"Sends the subvolume(s) specified by <subvol> to stdout." ,"By default, this will send the whole subvolume. To do an incremental" ,
@@ -707,5 +713,7 @@ const char * const cmd_send_usage[] = {" snapshot to send. This demands additional processing" ," (mostly IO bound) but is useful for the receive " ," command to report progress." , +"-a Use fallocate to pre-allocate file extents and to" , +" punch file holes, instead of writing zeroes to files." , NULL };
send-stream.c +13 -0
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->total_data_size (tmp, s->user);break ; +case BTRFS_SEND_C_FALLOCATE : + { +u32 flags; +u64 len; + +TLV_GET_STRING (s,BTRFS_SEND_A_PATH , &path); +TLV_GET_U32 (s,BTRFS_SEND_A_FALLOCATE_FLAGS , &flags); +TLV_GET_U64 (s,BTRFS_SEND_A_FILE_OFFSET , &offset); +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &len); + ret = s->ops->fallocate (path, flags, offset, len, + s->user); + } +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +2 -0
@@ -55,6 +55,8 @@ struct btrfs_send_ops { void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user);int (*total_data_size)(u64 size,void *user); +int (*fallocate)(const char *path,u32 flags,u64 offset, +u64 len,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 1 line
1.9.1
Re: [PATCH 3/4 v3] Btrfs-progs: send, implement fallocate command callback
Documentation/btrfs-send.txt An alternative would be to use pipes. · line 48
Quoted 9 lines · Filipe David Borba Manana
The fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
+-a::
Documentation/btrfs-send.txt +2
-0+Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes +to files.
(I wrote that in another mail and repeat it again in the context of the patch)
for v2 the fallocate mode should be default and IMHO not configurable by a flag at all - falloc is efficient and the (right) thing that the devs choose for the user.
I think that letting the user decide if he should use falloc or not is too low-level. From the UI perspective, specifying a protocol number should be enough.
There's currently no way to find the highest protocol version supported, but the sysfs flags can be enhanced with that.
Re: [PATCH 3/4 v3] Btrfs-progs: send, implement fallocate command callback
Quoted 13 lines · David Sterba
Documentation/btrfs-send.txt An alternative would be to use pipes. · line 48
Quoted 9 lines · Filipe David Borba Manana
The fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
+-a::
Documentation/btrfs-send.txt +2
-0+Use fallocate to pre-allocate file extents and to punch file holes, instead of writing zeroes +to files.(I wrote that in another mail and repeat it again in the context of the patch)
Quoted 7 lines
for v2 the fallocate mode should be default and IMHO not configurable by a flag at all - falloc is efficient and the (right) thing that the devs choose for the user.
I think that letting the user decide if he should use falloc or not is too low-level. From the UI perspective, specifying a protocol number should be enough.
Fine for me.
Quoted 3 lines
There's currently no way to find the highest protocol version supported, but the sysfs flags can be enhanced with that.
You mean, I guess, adding an entry named like/sys/fs/btrfs/features/send_stream_version
Thanks David
Signature · 5 lines
Filipe David Manana, "Reasonable men adapt themselves to the world. Unreasonable men adapt the world to themselves. That's why all progress depends on unreasonable men."
PATCH v4 3/4 Btrfs-progs: send, implement fallocate command callback
In reply to Filipe David Borba MananaThe fallocate send stream command, added in stream version 2, is used to pre-allocate space for files and punch file holes. This change implements the callback for that new command, using the fallocate function from the standard C library to carry out the specified action (allocate file space or punch a file hole).
---
V2: Use the new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE if the user
asks for it (-a command line option), which will make the kernel generate a version 2 send stream, so that old clients aren't affected.
V3: Rebased on new patchset (new version of patch 2/4).
V4: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and --stream-version=<version_number> respectively. Added commands for inode set flags and otime too.
cmds-receive.c | 38 ++++++++++++++++++++++++++++++++++++++ send-stream.c | 13 +++++++++++++ send-stream.h | 2 ++ 3 files changed, 53 insertions(+)
cmds-receive.c +38 -0
@@ -41,6 +41,7 @@#include <sys/types.h> #include <sys/xattr.h> #include <uuid/uuid.h> +#include <linux/falloc.h> #include "ctree.h" #include "ioctl.h"
@@ -887,6 +888,42 @@ out:return ret; } +static int process_fallocate (const char *path,u32 flags,u64 offset, +u64 len,void *user) +{ +struct btrfs_receive *r = user; +char *full_path =path_cat (r->full_subvol_path, path); +int mode =0 ; +int ret; + +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE ) + mode |=FALLOC_FL_KEEP_SIZE ; +if (flags &BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE ) + mode |=FALLOC_FL_PUNCH_HOLE ; + +if (g_verbose >=2 ) +fprintf (stderr, +"fallocate %s - flags %u, offset %llu, len %llu\n" , + path, flags, offset, len); + + ret =open_inode_for_write (r, full_path); +if (ret <0 ) + goto out; + + ret =fallocate (r->write_fd, mode, offset, len); +if (ret) { + ret = -errno; +fprintf (stderr, +"ERROR: fallocate against %s failed. %s\n" , + path,strerror (-ret)); + goto out; + } +update_progress (r, len); + +out: +free (full_path); +return ret; +}static struct btrfs_send_ops send_ops = { .subvol = process_subvol,
@@ -910,6 +947,7 @@ static struct btrfs_send_ops send_ops = { .chown = process_chown, .utimes = process_utimes, .total_data_size = process_total_data_size, + .fallocate = process_fallocate, };static int do_receive (struct btrfs_receive *r,const char *tomnt,int r_fd)
send-stream.c +13 -0
@@ -425,6 +425,19 @@ static int read_and_process_cmd(struct btrfs_send_stream *s)TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &tmp); ret = s->ops->total_data_size (tmp, s->user);break ; +case BTRFS_SEND_C_FALLOCATE : + { +u32 flags; +u64 len; + +TLV_GET_STRING (s,BTRFS_SEND_A_PATH , &path); +TLV_GET_U32 (s,BTRFS_SEND_A_FALLOCATE_FLAGS , &flags); +TLV_GET_U64 (s,BTRFS_SEND_A_FILE_OFFSET , &offset); +TLV_GET_U64 (s,BTRFS_SEND_A_SIZE , &len); + ret = s->ops->fallocate (path, flags, offset, len, + s->user); + } +break ;case BTRFS_SEND_C_END : ret =1 ;break ;
send-stream.h +2 -0
@@ -55,6 +55,8 @@ struct btrfs_send_ops { void *user);int (*update_extent)(const char *path,u64 offset,u64 len,void *user);int (*total_data_size)(u64 size,void *user); +int (*fallocate)(const char *path,u32 flags,u64 offset, +u64 len,void *user); }; int btrfs_read_and_process_send_stream(int fd,
Signature · 6 lines
1.9.1 -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo
PATCH 4/4 Btrfs-progs: add write and clone commands debug info to receive
In reply to Filipe David Borba MananaWhen specifying -vv print information about received write and clone commands too, as we do this for other commands already and it's very useful for debugging and troubleshooting.
--- cmds-receive.c | 9 +++++++++ 1 file changed, 9 insertions(+)
cmds-receive.c +9 -0
@@ -636,6 +636,10 @@ static int process_write(const char *path, const void *data, u64 offset,u64 pos =0 ;int w; +if (g_verbose >=2 ) +fprintf (stderr,"write %s, offset %llu, len %llu\n" , + path, offset, len); + ret =open_inode_for_write (r, full_path);if (ret <0 ) goto out;
@@ -672,6 +676,11 @@ static int process_clone(const char *path, u64 offset, u64 len,char *full_clone_path =NULL ;int clone_fd =-1 ; +if (g_verbose >=2 ) +fprintf (stderr, +"clone %s, offset %llu, len %llu, clone path %s, clone offset %llu\n" , + path, offset, len, clone_path, clone_offset); + ret =open_inode_for_write (r, full_path);if (ret <0 ) goto out;
Signature · 1 line
1.9.1
PATCH v2 4/4 Btrfs-progs: add write and clone commands debug info to receive
When specifying -vv print information about received write and clone commands too, as we do this for other commands already and it's very useful for debugging and troubleshooting.
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
cmds-receive.c | 9 +++++++++ 1 file changed, 9 insertions(+)
cmds-receive.c +9 -0
@@ -636,6 +636,10 @@ static int process_write(const char *path, const void *data, u64 offset,u64 pos =0 ;int w; +if (g_verbose >=2 ) +fprintf (stderr,"write %s, offset %llu, len %llu\n" , + path, offset, len); + ret =open_inode_for_write (r, full_path);if (ret <0 ) goto out;
@@ -672,6 +676,11 @@ static int process_clone(const char *path, u64 offset, u64 len,char *full_clone_path =NULL ;int clone_fd =-1 ; +if (g_verbose >=2 ) +fprintf (stderr, +"clone %s, offset %llu, len %llu, clone path %s, clone offset %llu\n" , + path, offset, len, clone_path, clone_offset); + ret =open_inode_for_write (r, full_path);if (ret <0 ) goto out;
Signature · 1 line
1.9.1
PATCH v2 1/4 Btrfs-progs: send, bump stream version
In reply to Filipe David Borba MananaThis increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files.
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
This doesn't break compatibility with older kernels or clients. In order to get a version 2 send stream, new flags must be passed to the send ioctl.
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
ioctl.h | 18 ++++++++++++++++++ send.h | 13 ++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-)
ioctl.h +18 -0
@@ -392,6 +392,24 @@ struct btrfs_ioctl_received_subvol_args {*/ #define BTRFS_SEND_FLAG_OMIT_END_CMD 0x4 +/* +* The sum of all length fields the receiver will get in write, clone and +* fallocate commands. +* This can be used by the receiver to compute progress, at the expense of some +* initial metadata scan performed by the sender (kernel). +* +* Added in send stream version 2. +*/ +#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE 0x8 + +/* +* Use fallocate command to pre-allocate file extents and punch file holes, +* instead of write commands with data buffers filled with 0 value bytes. +* +* Added in send stream version 2. +*/ +#define BTRFS_SEND_FLAG_SUPPORT_FALLOCATE 0x10 +struct btrfs_ioctl_send_args {__s64 send_fd;/* in */ __u64 clone_sources_count;/* in */
send.h +12 -1
@@ -24,7 +24,7 @@ extern "C" {#endif #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" -#define BTRFS_SEND_STREAM_VERSION 1 +#define BTRFS_SEND_STREAM_VERSION 2#define BTRFS_SEND_BUF_SIZE (1024 * 64)#define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +91,11 @@ enum btrfs_send_cmd {BTRFS_SEND_C_END ,BTRFS_SEND_C_UPDATE_EXTENT , + +/* added in stream version 2 */ +BTRFS_SEND_C_TOTAL_DATA_SIZE , +BTRFS_SEND_C_FALLOCATE , + __BTRFS_SEND_C_MAX, };#define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +134,16 @@ enum {BTRFS_SEND_A_CLONE_OFFSET ,BTRFS_SEND_A_CLONE_LEN , +/* added in stream version 2 */ +BTRFS_SEND_A_FALLOCATE_FLAGS , + __BTRFS_SEND_A_MAX, };#define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1) +#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE (1 << 0) +#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE (1 << 1) +#ifdef __KERNEL__long btrfs_ioctl_send (struct file *mnt_file,void __user *arg);#endif
Signature · 1 line
1.9.1
PATCH v3 1/4 Btrfs-progs: send, bump stream version
In reply to Filipe David Borba MananaThis increases the send stream version from version 1 to version 2, adding new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
2) fallocate - used to pre-allocate space for files and to punch holes in files;
3) inode set flags;
4) set inode otime.
This is preparation work for subsequent changes that implement the new features.
This doesn't break compatibility with older kernels or clients. In order to get a version 2 send stream, new flags must be passed to the send ioctl.
---
V2: Added new send ioctl flag BTRFS_SEND_FLAG_SUPPORT_FALLOCATE. A version 2
stream is now only produced is the ioctl caller specifies at least one of the new send flags (BTRFS_SEND_FLAG_SUPPORT_FALLOCATE or BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE).
V3: Removed BTRFS_SEND_FLAG_SUPPORT_FALLOCATE flag and -a command line option
for btrfs-send. Both were replaced with BTRFS_SEND_FLAG_STREAM_V2 and --stream-version=<version_number> respectively. Added commands for inode sets flags and otime too.
Documentation/btrfs-send.txt | 3 +++ cmds-send.c | 57 ++++++++++++++++++++++++++++++++++---------- ioctl.h | 15 ++++++++++++ send-stream.c | 2 +- send.h | 23 +++++++++++++++++- 5 files changed, 85 insertions(+), 15 deletions(-)
Documentation/btrfs-send.txt +3 -0
@@ -40,6 +40,9 @@ Use this snapshot as a clone source for an incremental send (multiple allowed). -f <outfile>:: Output is normally written to stdout. To write to a file, use this option. An alternative would be to use pipes. +--stream-version <version>:: +Ask the kernel to produce a specific send stream version. More recent stream versions provide +new features and better performance. Default value is 1. EXIT STATUS -----------
cmds-send.c +44 -13
104 lines not on this page · raw message
ioctl.h +15 -0
@@ -392,6 +392,21 @@ struct btrfs_ioctl_received_subvol_args {*/ #define BTRFS_SEND_FLAG_OMIT_END_CMD 0x4 +/* +* The sum of all length fields the receiver will get in write, clone and +* fallocate commands. +* This can be used by the receiver to compute progress, at the expense of some +* initial metadata scan performed by the sender (kernel). +* +* Added in send stream version 2. +*/ +#define BTRFS_SEND_FLAG_CALCULATE_DATA_SIZE 0x8 + +/* +* Used by a client to request a version 2 of the send stream. +*/ +#define BTRFS_SEND_FLAG_STREAM_V2 0x10 +struct btrfs_ioctl_send_args {__s64 send_fd;/* in */ __u64 clone_sources_count;/* in */
send-stream.c +1 -1
@@ -462,7 +462,7 @@ int btrfs_read_and_process_send_stream(int fd, } s.version =le32_to_cpu (hdr.version); -if (s.version >BTRFS_SEND_STREAM_VERSION ) { +if (s.version >BTRFS_SEND_STREAM_VERSION_MAX ) { ret = -EINVAL ;fprintf (stderr,"ERROR: Stream version %d not supported. " "Please upgrade btrfs-progs\n" , s.version);
send.h +22 -1
@@ -24,7 +24,10 @@ extern "C" {#endif #define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" -#define BTRFS_SEND_STREAM_VERSION 1 +#define BTRFS_SEND_STREAM_VERSION_1 1 +#define BTRFS_SEND_STREAM_VERSION_2 2 +/* Max supported stream version. */ +#define BTRFS_SEND_STREAM_VERSION_MAX BTRFS_SEND_STREAM_VERSION_2#define BTRFS_SEND_BUF_SIZE (1024 * 64)#define BTRFS_SEND_READ_SIZE (1024 * 48)
@@ -91,6 +94,15 @@ enum btrfs_send_cmd {BTRFS_SEND_C_END ,BTRFS_SEND_C_UPDATE_EXTENT , + +/* +* The following commands were added in stream version 2. +*/ +BTRFS_SEND_C_TOTAL_DATA_SIZE , +BTRFS_SEND_C_FALLOCATE , +BTRFS_SEND_C_INODE_SET_FLAGS , +BTRFS_SEND_C_UTIMES2 ,/* Same as UTIMES, but it includes OTIME too. */ + __BTRFS_SEND_C_MAX, };#define BTRFS_SEND_C_MAX (__BTRFS_SEND_C_MAX - 1)
@@ -129,10 +141,19 @@ enum {BTRFS_SEND_A_CLONE_OFFSET ,BTRFS_SEND_A_CLONE_LEN , +/* +* The following attributes were added in stream version 2. +*/ +BTRFS_SEND_A_FALLOCATE_FLAGS ,/* 32 bits */ +BTRFS_SEND_A_INODE_FLAGS ,/* 32 bits */ + __BTRFS_SEND_A_MAX, };#define BTRFS_SEND_A_MAX (__BTRFS_SEND_A_MAX - 1) +#define BTRFS_SEND_A_FALLOCATE_FLAG_KEEP_SIZE (1 << 0) +#define BTRFS_SEND_A_FALLOCATE_FLAG_PUNCH_HOLE (1 << 1) +#ifdef __KERNEL__long btrfs_ioctl_send (struct file *mnt_file,void __user *arg);#endif
Signature · 6 lines
1.9.1 -- To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo
Quoted 6 lines · Filipe David Borba Manana
This increases the send stream version from version 1 to version 2, adding 2 new commands:
1) total data size - used to tell the receiver how much file data the stream will add or update;
Quoted 6 lines
2) fallocate - used to pre-allocate space for files and to punch holes in files.
This is preparation work for subsequent changes that implement the new features (computing total data size and use fallocate for better performance).
Signed-off-by Filipe David Borba Manana <[email protected]>
The changes in the v2/3/4 look good, thanks. Patches added to next integratin.