gen_init_cpio: add copy_file_range support
cover gen_init_cpio: add copy_file_range support
This patchset adds copy_file_range() support to gen_init_cpio.
I've flagged it as RFC for a few reasons:
- more testing is needed
- it might not be useful for others
+ see message regarding reflink alignment requirements in PATCH 3/5
- I should probably provide some benchmark results to justify
David Disseldorp (5):
gen_init_cpio: write to fd instead of stdout stream
gen_init_cpio: support -o <output_path> parameter
gen_init_cpio: attempt copy_file_range for file data
gen_init_cpio: avoid duplicate strlen calls gen_initramfs.sh: use gen_init_cpio -o parameter
usr/gen_init_cpio.c | 192 +++++++++++++++++++++++++++---------------- usr/gen_initramfs.sh | 7 +- 2 files changed, 126 insertions(+), 73 deletions(-)
RFC PATCH 1/5 gen_init_cpio: write to fd instead of stdout stream
In preparation for more efficient archiving using copy_file_range(), switch from writing archive data to stdout to using STDOUT_FILENO and I/O via write(), dprintf(), etc.
Basic I/O error handling is added to cover cases such as ENOSPC. Partial writes are treated as errors.
--- usr/gen_init_cpio.c | 139 ++++++++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 58 deletions(-)
usr/gen_init_cpio.c +81 -58
@@ -23,64 +23,71 @@#define xstr (s) #s#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b)) +#define CPIO_HDR_LEN 110 +#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align)) +static char padding[512 ];static unsigned int offset;static unsigned int ino =721 ;static time_t default_mtime;static bool do_file_mtime;static bool do_csum = false; +static int outfd =STDOUT_FILENO ;struct file_handler {const char *type;int (*handler)(const char *line); }; -static void push_string (const char *name) +static int push_string (const char *name) {unsigned int name_len =strlen (name) +1 ; +ssize_t l; + + l =write (outfd, name, name_len); +if (l != name_len) +return -1 ; -fputs (name, stdout); -putchar (0 ); offset += name_len; +return 0 ; } -static void push_pad (void ) +static int push_pad (size_t padlen) { -while (offset &3 ) { -putchar (0 ); - offset++; - } +ssize_t l =0 ; + +if (!padlen) +return 0 ; + +if (padlen <sizeof (padding)) + l =write (outfd, padding, padlen); +if (l != padlen) +return -1 ; + + offset += padlen; +return 0 ; } -static void push_rest (const char *name) +static int push_rest (const char *name) {unsigned int name_len =strlen (name) +1 ; -unsigned int tmp_ofs; +ssize_t l; -fputs (name, stdout); -putchar (0 ); - offset += name_len; + l =write (outfd, name, name_len); +if (l != name_len) +return -1 ; - tmp_ofs = name_len +110 ; -while (tmp_ofs &3 ) { -putchar (0 ); - offset++; - tmp_ofs++; - } -} + offset += name_len; -static void push_hdr (const char *s) -{ -fputs (s, stdout); - offset +=110 ; +return push_pad (padlen (name_len +CPIO_HDR_LEN ,4 )); } -static void cpio_trailer (void ) +static int cpio_trailer (void ) { -char s[256 ];const char name[] ="TRAILER!!!" ; +int l; - sprintf(s, "%s%08X%08X%08lX%08lX%08X%08lX" + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ 0, /* ino */
@@ -96,23 +103,24 @@ static void cpio_trailer(void)0 ,/* rminor */ (unsigned )strlen (name)+1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += l; -while (offset %512 ) { -putchar (0 ); - offset++; - } +if (l !=CPIO_HDR_LEN + ||push_rest (name) <0 + ||push_pad (padlen (offset,512 )) <0 ) +return -1 ; + +return 0 ; }static int cpio_mkslink (const char *name,const char *target,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int l;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -128,12 +136,17 @@ static int cpio_mkslink(const char *name, const char *target,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); -push_string (target); -push_pad (); + offset += l; + +if (l !=CPIO_HDR_LEN + ||push_string (name) <0 + ||push_pad (padlen (offset,4 )) <0 + ||push_string (target) <0 + ||push_pad (padlen (offset,4 )) <0 ) +return -1 ; +return 0 ; + }static int cpio_mkslink_line (const char *line)
@@ -157,11 +170,11 @@ static int cpio_mkslink_line(const char *line)static int cpio_mkgeneric (const char *name,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int l;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -177,8 +190,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += l; + +if (l !=CPIO_HDR_LEN + ||push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -246,7 +263,7 @@ static int cpio_mknod(const char *name, unsigned int mode, uid_t uid, gid_t gid, char dev_type, unsigned int maj, unsigned int min) { - char s[256]; + int l;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -255,7 +272,7 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -271,8 +288,12 @@ static int cpio_mknod(const char *name, unsigned int mode, min,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += l; + +if (l !=CPIO_HDR_LEN + ||push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -324,11 +345,9 @@ static int cpio_mkfile(const char *name, const char *location, unsigned int mode, uid_t uid, gid_t gid, unsigned int nlinks) { - char s[256]; struct stat buf;unsigned long size; -int file; -int retval; +int file, retval, l;int rc =-1 ;time_t mtime;int namesize;
@@ -386,7 +405,7 @@ static int cpio_mkfile(const char *name, const char *location,if (name[0 ] =='/' ) name++; namesize =strlen (name) +1 ; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08lX%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino, /* ino */
@@ -402,9 +421,12 @@ static int cpio_mkfile(const char *name, const char *location,0 ,/* rminor */ namesize,/* namesize */ size ? csum : 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); + offset += l; + +if (l !=CPIO_HDR_LEN + ||push_string (name) <0 + ||push_pad (padlen (offset,4 )) <0 ) + goto error;while (size) {unsigned char filebuf[65536 ];
@@ -417,14 +439,15 @@ static int cpio_mkfile(const char *name, const char *location, goto error; } -if (fwrite (filebuf, this_read,1 , stdout) !=1 ) { +if (write (outfd, filebuf, this_read) != this_read) {fprintf (stderr,"writing filebuf failed\n" ); goto error; } offset += this_read; size -= this_read; } -push_pad (); +if (push_pad (padlen (offset,4 )) <0 ) + goto error; name += namesize; }
@@ -691,7 +714,7 @@ int main (int argc, char *argv[]) } }if (ec ==0 ) -cpio_trailer (); + ec =cpio_trailer ();exit (ec); }
Signature · 1 line
2.43.0
RFC PATCH 2/5 gen_init_cpio: support -o <output_path> parameter
In reply to David DisseldorpThis is another preparatory change to allow for reflink-optimized cpio archives with file data written / cloned via copy_file_range(). The output file is truncated prior to write, so that it maps to usr/gen_initramfs.sh usage. It may make sense to offer an append option in future, for easier archive concatenation.
--- usr/gen_init_cpio.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-)
usr/gen_init_cpio.c +15 -4
@@ -110,7 +110,7 @@ static int cpio_trailer(void) || push_pad(padlen(offset, 512)) < 0)return -1 ; -return 0 ; +return fsync (outfd); } static int cpio_mkslink(const char *name, const char *target,
@@ -532,7 +532,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] <cpio_list>\n" +"\t%s [-t <timestamp>] [-c] [-o <output_path>] <cpio_list>\n" "\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n"
@@ -569,7 +569,8 @@ static void usage(const char *prog)"as mtime for symlinks, directories, regular and special files.\n" "The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" -"-c: calculate and store 32-bit checksums for file data.\n" , +"-c: calculate and store 32-bit checksums for file data.\n" +"<output_path>: write cpio to this file instead of stdout\n" , prog); }
@@ -611,7 +612,7 @@ int main (int argc, char *argv[]) default_mtime =time (NULL );while (1 ) { -int opt =getopt (argc, argv,"t:ch" ); +int opt =getopt (argc, argv,"t:cho:" );char *invalid;if (opt ==-1 )
@@ -630,6 +631,16 @@ int main (int argc, char *argv[])case 'c' : do_csum = true;break ; +case 'o' : + outfd =open (optarg, +O_WRONLY |O_CREAT |O_LARGEFILE |O_TRUNC , +0600 ); +if (outfd <0 ) { +fprintf (stderr,"failed to open %s\n" , optarg); +usage (argv[0 ]); +exit (1 ); + } +break ;case 'h' :case '?' :usage (argv[0 ]);
Signature · 1 line
2.43.0
RFC PATCH 3/5 gen_init_cpio: attempt copy_file_range for file data
In reply to David DisseldorpThe copy_file_range() syscall can improve copy performance by cloning extents between cpio archive source and destination files. Existing read / write based copy logic is retained for fallback in case the copy_file_range() syscall is unsupported or unavailable due to cross-filesystem EXDEV, etc.
Clone, as opposed to copy, of source file extents into the output cpio archive may (e.g. on Btrfs) require alignment of the output to the filesystem block size. This could be achieved by inserting padding entries into the cpio archive manifest. In future it may be worth zero-padding the filename field similar to dracut-cpio[1].
--- usr/gen_init_cpio.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
usr/gen_init_cpio.c +11 -1
@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE#include <stdio.h> #include <stdlib.h> #include <stdint.h>
@@ -353,6 +354,7 @@ static int cpio_mkfile(const char *name, const char *location,int namesize;unsigned int i;uint32_t csum =0 ; +ssize_t this_read; mode |=S_IFREG ;
@@ -428,9 +430,17 @@ static int cpio_mkfile(const char *name, const char *location, || push_pad(padlen(offset, 4)) < 0) goto error; +if (size) { + this_read =copy_file_range (file,NULL , outfd,NULL , size,0 ); +if (this_read >0 && this_read <= size) { + offset += this_read; + size -= this_read; + } +/* short or failed copy falls back to read/write... */ + } +while (size) {unsigned char filebuf[65536 ]; -ssize_t this_read;size_t this_size =MIN (size,sizeof (filebuf)); this_read =read (file, filebuf, this_size);
Signature · 1 line
2.43.0
RFC PATCH 4/5 gen_init_cpio: avoid duplicate strlen calls
In reply to David DisseldorpWe determine the filename length for the cpio header, so shouldn't recalculate it when writing out the filename.
--- usr/gen_init_cpio.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-)
usr/gen_init_cpio.c +24 -16
@@ -25,6 +25,7 @@#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b))#define CPIO_HDR_LEN 110 +#define CPIO_TRAILER "TRAILER!!!"#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align))static char padding[512 ];
@@ -40,9 +41,8 @@ struct file_handler {int (*handler)(const char *line); }; -static int push_string (const char *name) +static int push_buf (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t l; l =write (outfd, name, name_len);
@@ -69,9 +69,8 @@ static int push_pad(size_t padlen)return 0 ; } -static int push_rest (const char *name) +static int push_rest (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t l; l =write (outfd, name, name_len);
@@ -85,8 +84,8 @@ static int push_rest(const char *name)static int cpio_trailer (void ) { -const char name[] ="TRAILER!!!" ;int l; +unsigned int namesize =sizeof (CPIO_TRAILER ); l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X",
@@ -102,12 +101,12 @@ static int cpio_trailer(void)0 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name)+1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += l;if (l !=CPIO_HDR_LEN - ||push_rest (name) <0 + ||push_rest (CPIO_TRAILER , namesize) <0 ||push_pad (padlen (offset,512 )) <0 )return -1 ;
@@ -118,9 +117,12 @@ static int cpio_mkslink(const char *name, const char *target, unsigned int mode, uid_t uid, gid_t gid) { int l; +unsigned int namesize, targetsize =strlen (target) +1 ;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -130,19 +132,19 @@ static int cpio_mkslink(const char *name, const char *target, (long ) gid,/* gid */ 1 ,/* nlink */ (long ) default_mtime,/* mtime */ - (unsigned )strlen (target)+1 ,/* filesize */ + targetsize,/* filesize */ 3 ,/* major */ 1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += l;if (l !=CPIO_HDR_LEN - ||push_string (name) <0 + ||push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 - ||push_string (target) <0 + ||push_buf (target, targetsize) <0 ||push_pad (padlen (offset,4 )) <0 )return -1 ;
@@ -172,9 +174,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode, uid_t uid, gid_t gid) { int l; +unsigned int namesize;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -189,12 +194,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += l;if (l !=CPIO_HDR_LEN - ||push_rest (name) <0 ) + ||push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -265,6 +270,7 @@ static int cpio_mknod(const char *name, unsigned int mode, unsigned int maj, unsigned int min) { int l; +unsigned int namesize;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -273,6 +279,8 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -287,12 +295,12 @@ static int cpio_mknod(const char *name, unsigned int mode,1 ,/* minor */ maj,/* rmajor */ min,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += l;if (l !=CPIO_HDR_LEN - ||push_rest (name) <0 ) + ||push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -426,7 +434,7 @@ static int cpio_mkfile(const char *name, const char *location, offset += l;if (l !=CPIO_HDR_LEN - ||push_string (name) <0 + ||push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 ) goto error;
Signature · 1 line
2.43.0
RFC PATCH 5/5 gen_initramfs.sh: use gen_init_cpio -o parameter
In reply to David Disseldorpgen_init_cpio can now write to a file directly, so use it when gen_initramfs.sh is called with -o (e.g. usr/Makefile invocation).
--- usr/gen_initramfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
usr/gen_initramfs.sh +4 -3
@@ -193,7 +193,8 @@ root_gid=0 dep_list= timestamp= cpio_list=$(mktemp ${TMPDIR:-/tmp}/cpiolist.XXXXXX) -output="/dev/stdout" +# gen_init_cpio writes to stdout by default +output="" trap "rm -f $cpio_list" EXIT
@@ -207,7 +208,7 @@ while [ $# -gt 0 ]; do shift ;; "-o") # generate cpio image named $1 - output="$1" + output="-o $1" shift ;; "-u") # map $1 to uid=0 (root)
@@ -246,4 +247,4 @@ done # If output_file is set we will generate cpio archive # we are careful to delete tmp files -usr/gen_init_cpio $timestamp $cpio_list > $output +usr/gen_init_cpio $output $timestamp $cpio_list
Signature · 1 line
2.43.0