gen_init_cpio: add copy_file_range / reflink support
Subsystems (4)
v2 cover gen_init_cpio: add copy_file_range / reflink support
This patchset adds copy_file_range() support to gen_init_cpio. When combined with data segment alignment, large-file archiving performance is improved on Btrfs and XFS due to reflinks (see 7/7 benchmarks).
cpio data segment alignment is provided by "bending" the newc spec to zero-pad the filename field. GNU cpio and Linux initramfs extractors handle this fine as long as PATH_MAX isn't exceeded.
Changes since v1 RFC
- add alignment patches 6-7
- slightly rework commit and error messages
- rename l->len to avoid 1/i confusion
David Disseldorp (7):
gen_init_cpio: write to fd instead of stdout stream
gen_init_cpio: support -o <output_path> parameter
gen_init_cpio: attempt copy_file_range for file data
gen_init_cpio: avoid duplicate strlen calls gen_initramfs.sh: use gen_init_cpio -o parameter
docs: initramfs: file data alignment via name padding
gen_init_cpio: add -a <data_align> as reflink optimization
.../driver-api/early-userspace/buffer-format.rst | 5 + usr/gen_init_cpio.c | 234 ++++++++++++++------- usr/gen_initramfs.sh | 7 +- 3 files changed, 166 insertions(+), 80 deletions(-)
PATCH v2 1/7 gen_init_cpio: write to fd instead of stdout stream
In preparation for more efficient archiving using copy_file_range(), switch from writing archive data to stdout to using STDOUT_FILENO and I/O via write(), dprintf(), etc.
Basic I/O error handling is added to cover cases such as ENOSPC. Partial writes are treated as errors.
--- usr/gen_init_cpio.c | 139 ++++++++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 58 deletions(-)
usr/gen_init_cpio.c +81 -58
@@ -23,64 +23,71 @@#define xstr (s) #s#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b)) +#define CPIO_HDR_LEN 110 +#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align)) +static char padding[512 ];static unsigned int offset;static unsigned int ino =721 ;static time_t default_mtime;static bool do_file_mtime;static bool do_csum = false; +static int outfd =STDOUT_FILENO ;struct file_handler {const char *type;int (*handler)(const char *line); }; -static void push_string (const char *name) +static int push_string (const char *name) {unsigned int name_len =strlen (name) +1 ; +ssize_t len; + + len =write (outfd, name, name_len); +if (len != name_len) +return -1 ; -fputs (name, stdout); -putchar (0 ); offset += name_len; +return 0 ; } -static void push_pad (void ) +static int push_pad (size_t padlen) { -while (offset &3 ) { -putchar (0 ); - offset++; - } +ssize_t len =0 ; + +if (!padlen) +return 0 ; + +if (padlen <sizeof (padding)) + len =write (outfd, padding, padlen); +if (len != padlen) +return -1 ; + + offset += padlen; +return 0 ; } -static void push_rest (const char *name) +static int push_rest (const char *name) {unsigned int name_len =strlen (name) +1 ; -unsigned int tmp_ofs; +ssize_t len; -fputs (name, stdout); -putchar (0 ); - offset += name_len; + len =write (outfd, name, name_len); +if (len != name_len) +return -1 ; - tmp_ofs = name_len +110 ; -while (tmp_ofs &3 ) { -putchar (0 ); - offset++; - tmp_ofs++; - } -} + offset += name_len; -static void push_hdr (const char *s) -{ -fputs (s, stdout); - offset +=110 ; +return push_pad (padlen (name_len +CPIO_HDR_LEN ,4 )); } -static void cpio_trailer (void ) +static int cpio_trailer (void ) { -char s[256 ];const char name[] ="TRAILER!!!" ; +int len; - sprintf(s, "%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ 0, /* ino */
@@ -96,23 +103,24 @@ static void cpio_trailer(void)0 ,/* rminor */ (unsigned )strlen (name)+1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; -while (offset %512 ) { -putchar (0 ); - offset++; - } +if (len !=CPIO_HDR_LEN + ||push_rest (name) <0 + ||push_pad (padlen (offset,512 )) <0 ) +return -1 ; + +return 0 ; }static int cpio_mkslink (const char *name,const char *target,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int len;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -128,12 +136,17 @@ static int cpio_mkslink(const char *name, const char *target,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); -push_string (target); -push_pad (); + offset += len; + +if (len !=CPIO_HDR_LEN + ||push_string (name) <0 + ||push_pad (padlen (offset,4 )) <0 + ||push_string (target) <0 + ||push_pad (padlen (offset,4 )) <0 ) +return -1 ; +return 0 ; + }static int cpio_mkslink_line (const char *line)
@@ -157,11 +170,11 @@ static int cpio_mkslink_line(const char *line)static int cpio_mkgeneric (const char *name,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int len;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -177,8 +190,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; + +if (len !=CPIO_HDR_LEN + ||push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -246,7 +263,7 @@ static int cpio_mknod(const char *name, unsigned int mode, uid_t uid, gid_t gid, char dev_type, unsigned int maj, unsigned int min) { - char s[256]; + int len;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -255,7 +272,7 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -271,8 +288,12 @@ static int cpio_mknod(const char *name, unsigned int mode, min,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; + +if (len !=CPIO_HDR_LEN + ||push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -324,11 +345,9 @@ static int cpio_mkfile(const char *name, const char *location, unsigned int mode, uid_t uid, gid_t gid, unsigned int nlinks) { - char s[256]; struct stat buf;unsigned long size; -int file; -int retval; +int file, retval, len;int rc =-1 ;time_t mtime;int namesize;
@@ -386,7 +405,7 @@ static int cpio_mkfile(const char *name, const char *location,if (name[0 ] =='/' ) name++; namesize =strlen (name) +1 ; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08lX%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino, /* ino */
@@ -402,9 +421,12 @@ static int cpio_mkfile(const char *name, const char *location,0 ,/* rminor */ namesize,/* namesize */ size ? csum : 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); + offset += len; + +if (len !=CPIO_HDR_LEN + ||push_string (name) <0 + ||push_pad (padlen (offset,4 )) <0 ) + goto error;while (size) {unsigned char filebuf[65536 ];
@@ -417,14 +439,15 @@ static int cpio_mkfile(const char *name, const char *location, goto error; } -if (fwrite (filebuf, this_read,1 , stdout) !=1 ) { +if (write (outfd, filebuf, this_read) != this_read) {fprintf (stderr,"writing filebuf failed\n" ); goto error; } offset += this_read; size -= this_read; } -push_pad (); +if (push_pad (padlen (offset,4 )) <0 ) + goto error; name += namesize; }
@@ -691,7 +714,7 @@ int main (int argc, char *argv[]) } }if (ec ==0 ) -cpio_trailer (); + ec =cpio_trailer ();exit (ec); }
Signature · 1 line
2.43.0
Nicolas Schier
maintainerRe: [PATCH v2 1/7] gen_init_cpio: write to fd instead of stdout stream
Quoted 15 lines · David Disseldorp
In preparation for more efficient archiving using
copy_file_range(), switch from writing archive data to stdout to usingSTDOUT_FILENOand I/O viawrite(),dprintf(), etc.
Basic I/O error handling is added to cover cases such as ENOSPC. Partial writes are treated as errors.Signed-off-by David Disseldorp <[email protected]>--- usr/gen_init_cpio.c | 139 ++++++++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 58 deletions(-)usr/gen_init_cpio.c +0
-0
[...]
Quoted 5 lines
usr/gen_init_cpio.c +0
-1@@ -96,23 +103,24 @@ static void cpio_trailer(void)0 ,/* rminor */ (unsigned )strlen (name)+1 ,/* namesize */ 0); /* chksum */ -push_hdr (s);
usr/gen_init_cpio.c static void cpio_trailer(void) · line 110
Quoted 10 lines
usr/gen_init_cpio.c +4
-5-push_rest (name); + offset += len; -while (offset %512 ) { -putchar (0 ); - offset++; - } +if (len !=CPIO_HDR_LEN + ||push_rest (name) <0 + ||push_pad (padlen (offset,512 )) <0 )
Thanks, patch looks good to me.
Just a minor coding style bike shedding: Starting continuation lines with '||' seems to be rather unusual in Linux code:
$ git grep -Ee '^\s*(\|\||&&)' **/*.c | wc -l
64
$ git grep -Ee '(\|\||&&)$' **/*.c | wc -l
2553Kind regards,
Nicolas
PATCH v2 2/7 gen_init_cpio: support -o <output_path> parameter
In reply to David DisseldorpThis is another preparatory change to allow for reflink-optimized cpio archives with file data written / cloned via copy_file_range(). The output file is truncated prior to write, so that it maps to usr/gen_initramfs.sh usage. It may make sense to offer an append option in future, for easier archive concatenation.
--- usr/gen_init_cpio.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-)
usr/gen_init_cpio.c +15 -4
@@ -110,7 +110,7 @@ static int cpio_trailer(void) || push_pad(padlen(offset, 512)) < 0)return -1 ; -return 0 ; +return fsync (outfd); } static int cpio_mkslink(const char *name, const char *target,
@@ -532,7 +532,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] <cpio_list>\n" +"\t%s [-t <timestamp>] [-c] [-o <output_path>] <cpio_list>\n" "\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n"
@@ -569,7 +569,8 @@ static void usage(const char *prog)"as mtime for symlinks, directories, regular and special files.\n" "The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" -"-c: calculate and store 32-bit checksums for file data.\n" , +"-c: calculate and store 32-bit checksums for file data.\n" +"<output_path>: write cpio to this file instead of stdout\n" , prog); }
@@ -611,7 +612,7 @@ int main (int argc, char *argv[]) default_mtime =time (NULL );while (1 ) { -int opt =getopt (argc, argv,"t:ch" ); +int opt =getopt (argc, argv,"t:cho:" );char *invalid;if (opt ==-1 )
@@ -630,6 +631,16 @@ int main (int argc, char *argv[])case 'c' : do_csum = true;break ; +case 'o' : + outfd =open (optarg, +O_WRONLY |O_CREAT |O_LARGEFILE |O_TRUNC , +0600 ); +if (outfd <0 ) { +fprintf (stderr,"failed to open %s\n" , optarg); +usage (argv[0 ]); +exit (1 ); + } +break ;case 'h' :case '?' :usage (argv[0 ]);
Signature · 1 line
2.43.0
Nicolas Schier
maintainerRe: [PATCH v2 2/7] gen_init_cpio: support -o <output_path> parameter
Quoted 34 lines · David Disseldorp
This is another preparatory change to allow for reflink-optimized cpio archives with file data written / cloned via
copy_file_range(). The output file is truncated prior to write, so that it maps tousr/gen_initramfs.shusage. It may make sense to offer an append option in future, for easier archive concatenation.Signed-off-by David Disseldorp <[email protected]>--- usr/gen_init_cpio.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-)usr/gen_init_cpio.c +2
-2@@ -110,7 +110,7 @@ static int cpio_trailer(void) || push_pad(padlen(offset, 512)) < 0)return -1 ; -return 0 ; +return fsync (outfd); } static int cpio_mkslink(const char *name, const char *target,@@ -532,7 +532,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] <cpio_list>\n" +"\t%s [-t <timestamp>] [-c] [-o <output_path>] <cpio_list>\n" "\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n"
usr/gen_init_cpio.c static void usage(const char *prog) · line 573
Quoted 6 lines
usr/gen_init_cpio.c +2
-1@@ -569,7 +569,8 @@ static void usage(const char *prog)"as mtime for symlinks, directories, regular and special files.\n" "The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" - "-c: calculate and store 32-bit checksums for file data.\n", +"-c: calculate and store 32-bit checksums for file data.\n" + "<output_path>: write cpio to this file instead of stdout\n",
gen_init_cpio writes only a single output file (instead of multiple files to an output directory), I'd suggest to name the parameter just 'file' or 'output_file'.
I'd like to see the the '... -o ...' patch right after this one.
For compilability, I had to add
#define _LARGEFILE64_SOURCE
or
#define _GNU_SOURCE
as you do in the next patch, in order to get O_LARGEFILE defined.
Kind regards,
Nicolas
PATCH v2 3/7 gen_init_cpio: attempt copy_file_range for file data
In reply to David DisseldorpThe copy_file_range syscall can improve copy performance by cloning extents between cpio archive source and destination files. Existing read / write based copy logic is retained for fallback in case the copy_file_range syscall is unsupported or unavailable due to cross-filesystem EXDEV, etc.
Clone or reflink, as opposed to copy, of source file extents into the output cpio archive may (e.g. on Btrfs and XFS) require alignment of the output to the filesystem block size. This could be achieved by inserting padding entries into the cpio archive manifest.
--- usr/gen_init_cpio.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
usr/gen_init_cpio.c +13 -1
@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE#include <stdio.h> #include <stdlib.h> #include <stdint.h>
@@ -353,6 +354,7 @@ static int cpio_mkfile(const char *name, const char *location,int namesize;unsigned int i;uint32_t csum =0 ; +ssize_t this_read; mode |=S_IFREG ;
@@ -428,9 +430,19 @@ static int cpio_mkfile(const char *name, const char *location, || push_pad(padlen(offset, 4)) < 0) goto error; +if (size) { + this_read =copy_file_range (file,NULL , outfd,NULL , size,0 ); +if (this_read >0 ) { +if (this_read > size) + goto error; + offset += this_read; + size -= this_read; + } +/* short or failed copy falls back to read/write... */ + } +while (size) {unsigned char filebuf[65536 ]; -ssize_t this_read;size_t this_size =MIN (size,sizeof (filebuf)); this_read =read (file, filebuf, this_size);
Signature · 1 line
2.43.0
Nicolas Schier
maintainerRe: [PATCH v2 3/7] gen_init_cpio: attempt copy_file_range for file data
Quoted 11 lines · David Disseldorp
The
copy_file_rangesyscall can improve copy performance by cloning extents between cpio archive source and destination files. Existing read / write based copy logic is retained for fallback in case thecopy_file_rangesyscall is unsupported or unavailable due to cross-filesystem EXDEV, etc.Clone or reflink, as opposed to copy, of source file extents into the output cpio archive may (e.g. on Btrfs and XFS) require alignment of the output to the filesystem block size. This could be achieved by inserting padding entries into the cpio archive manifest.
Quoted 4 lines
Signed-off-by David Disseldorp <[email protected]>--- usr/gen_init_cpio.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
Thanks. I like introducing copy_file_range() here, it reduces the cpio generation time on my system by a about 30% on a btrfs filesystem. May cpio_mkfile_csum() now the slowest part?
Kind regards,
Nicolas
Re: [PATCH v2 3/7] gen_init_cpio: attempt copy_file_range for file data
On Mon, 18 Aug 2025 20:40:39 +0200, Nicolas Schier wrote:
...
Quoted 3 lines
Thanks. I like introducing
copy_file_range()here, it reduces the cpio generation time on my system by a about 30% on a btrfs filesystem. Maycpio_mkfile_csum()now the slowest part?
cpio checksums are slow, but I don't think it's worth optimizing for at this stage: they're optional (-c) and support for them was only recently added (800c24dc34b93). Besides, there are better ways to guarantee initramfs integrity.
Thanks, David
PATCH v2 4/7 gen_init_cpio: avoid duplicate strlen calls
In reply to David DisseldorpWe determine the filename length for the cpio header, so shouldn't recalculate it when writing out the filename.
--- usr/gen_init_cpio.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-)
usr/gen_init_cpio.c +24 -16
@@ -25,6 +25,7 @@#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b))#define CPIO_HDR_LEN 110 +#define CPIO_TRAILER "TRAILER!!!"#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align))static char padding[512 ];
@@ -40,9 +41,8 @@ struct file_handler {int (*handler)(const char *line); }; -static int push_string (const char *name) +static int push_buf (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t len; len =write (outfd, name, name_len);
@@ -69,9 +69,8 @@ static int push_pad(size_t padlen)return 0 ; } -static int push_rest (const char *name) +static int push_rest (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t len; len =write (outfd, name, name_len);
@@ -85,8 +84,8 @@ static int push_rest(const char *name)static int cpio_trailer (void ) { -const char name[] ="TRAILER!!!" ;int len; +unsigned int namesize =sizeof (CPIO_TRAILER ); len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X",
@@ -102,12 +101,12 @@ static int cpio_trailer(void)0 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name)+1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN - ||push_rest (name) <0 + ||push_rest (CPIO_TRAILER , namesize) <0 ||push_pad (padlen (offset,512 )) <0 )return -1 ;
@@ -118,9 +117,12 @@ static int cpio_mkslink(const char *name, const char *target, unsigned int mode, uid_t uid, gid_t gid) { int len; +unsigned int namesize, targetsize =strlen (target) +1 ;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -130,19 +132,19 @@ static int cpio_mkslink(const char *name, const char *target, (long ) gid,/* gid */ 1 ,/* nlink */ (long ) default_mtime,/* mtime */ - (unsigned )strlen (target)+1 ,/* filesize */ + targetsize,/* filesize */ 3 ,/* major */ 1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN - ||push_string (name) <0 + ||push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 - ||push_string (target) <0 + ||push_buf (target, targetsize) <0 ||push_pad (padlen (offset,4 )) <0 )return -1 ;
@@ -172,9 +174,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode, uid_t uid, gid_t gid) { int len; +unsigned int namesize;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -189,12 +194,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN - ||push_rest (name) <0 ) + ||push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -265,6 +270,7 @@ static int cpio_mknod(const char *name, unsigned int mode, unsigned int maj, unsigned int min) { int len; +unsigned int namesize;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -273,6 +279,8 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -287,12 +295,12 @@ static int cpio_mknod(const char *name, unsigned int mode,1 ,/* minor */ maj,/* rmajor */ min,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN - ||push_rest (name) <0 ) + ||push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -426,7 +434,7 @@ static int cpio_mkfile(const char *name, const char *location, offset += len;if (len !=CPIO_HDR_LEN - ||push_string (name) <0 + ||push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 ) goto error;
Signature · 1 line
2.43.0
PATCH v2 5/7 gen_initramfs.sh: use gen_init_cpio -o parameter
In reply to David Disseldorpgen_init_cpio can now write to a file directly, so use it when gen_initramfs.sh is called with -o (e.g. usr/Makefile invocation).
--- usr/gen_initramfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
usr/gen_initramfs.sh +4 -3
@@ -193,7 +193,8 @@ root_gid=0 dep_list= timestamp= cpio_list=$(mktemp ${TMPDIR:-/tmp}/cpiolist.XXXXXX) -output="/dev/stdout" +# gen_init_cpio writes to stdout by default +output="" trap "rm -f $cpio_list" EXIT
@@ -207,7 +208,7 @@ while [ $# -gt 0 ]; do shift ;; "-o") # generate cpio image named $1 - output="$1" + output="-o $1" shift ;; "-u") # map $1 to uid=0 (root)
@@ -246,4 +247,4 @@ done # If output_file is set we will generate cpio archive # we are careful to delete tmp files -usr/gen_init_cpio $timestamp $cpio_list > $output +usr/gen_init_cpio $output $timestamp $cpio_list
Signature · 1 line
2.43.0
Nicolas Schier
maintainerRe: [PATCH v2 5/7] gen_initramfs.sh: use gen_init_cpio -o parameter
Quoted 32 lines · David Disseldorp
gen_init_cpiocan now write to a file directly, so use it whengen_initramfs.shis called with -o (e.g. usr/Makefile invocation).Signed-off-by David Disseldorp <[email protected]>--- usr/gen_initramfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)usr/gen_initramfs.sh +3
-2@@ -193,7 +193,8 @@ root_gid=0 dep_list= timestamp= cpio_list=$(mktemp ${TMPDIR:-/tmp}/cpiolist.XXXXXX) -output="/dev/stdout" +# gen_init_cpio writes to stdout by default +output="" trap "rm -f $cpio_list" EXIT@@ -207,7 +208,7 @@ while [ $# -gt 0 ]; do shift ;; "-o") # generate cpio image named $1 - output="$1" + output="-o $1" shift ;; "-u") # map $1 to uid=0 (root)
usr/gen_initramfs.sh done · line 250
Quoted 5 lines
usr/gen_initramfs.sh +1
-1@@ -246,4 +247,4 @@ done # If output_file is set we will generate cpio archive # we are careful to delete tmp files -usr/gen_init_cpio $timestamp $cpio_list > $output +usr/gen_init_cpio $output $timestamp $cpio_list
I think it would have been sufficient to replace '> $output' by '-o $output'.
PATCH v2 6/7 docs: initramfs: file data alignment via name padding
In reply to David DisseldorpThe existing cpio extraction logic reads (maximum PATH_MAX) name_len bytes from the archive into the collected name buffer and ensures that the trailing byte is a null-terminator. This allows the actual file name to be shorter than name_len, with the name string simply zero-terminated prior to the last byte.
Initramfs generators, such as dracut-cpio[1], can take advantage of name zero-padding to align file data segments within the archive to filesystem block boundaries. Block boundary alignment may allow the copy_file_range syscall to reflink archive source and destination extents.
--- Documentation/driver-api/early-userspace/buffer-format.rst | 5 +++++ 1 file changed, 5 insertions(+)
Documentation/driver-api/early-userspace/buffer-format.rst +5 -0
@@ -86,6 +86,11 @@ c_mtime is ignored unless CONFIG_INITRAMFS_PRESERVE_MTIME=y is set. The c_filesize should be zero for any file which is not a regular file or symlink. +c_namesize may account for more than one trailing '\0', as long as the +value doesn't exceed PATH_MAX. This can be useful for ensuring that a +subsequent file data segment is aligned, e.g. to a filesystem block +boundary. + The c_chksum field contains a simple 32-bit unsigned sum of all the bytes in the data field. cpio(1) refers to this as "crc", which is clearly incorrect (a cyclic redundancy check is a different and
Signature · 1 line
2.43.0
PATCH v2 7/7 gen_init_cpio: add -a <data_align> as reflink optimization
In reply to David DisseldorpAs described in buffer-format.rst, the existing initramfs.c extraction logic works fine if the cpio filename field is padded out with trailing zeros, with a caveat that the padded namesize can't exceed PATH_MAX.
Add filename zero-padding logic to gen_init_cpio, which can be triggered via the new -a <data_align> parameter. Performance and storage utilization is improved for Btrfs and XFS workloads, as copy_file_range can reflink the entire source file into a filesystem block-size aligned destination offset within the cpio archive.
Btrfs benchmarks run on 6.15.8-1-default (Tumbleweed) x86_64 host:
Quoted 2 lines
truncate --size=2G
/tmp/backing.img/sbin/mkfs.btrfs/tmp/backing.img
...
Sector size: 4096 (CPU page size: 4096)
...
Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mnt
mnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10 gen_init_cpio -o unaligned_btrfs list
...
0.023496 +- 0.000472 seconds time elapsed ( +- 2.01% )
mnt> perf stat -r 10 gen_init_cpio -o aligned_btrfs -a 4096 list
...
0.0010010 +- 0.0000565 seconds time elapsed ( +- 5.65% )
mnt> /sbin/xfs_io -c "fiemap -v" unaligned_btrfs unaligned_btrfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..40967]: 695040..736007 40968 0x1
mnt> /sbin/xfs_io -c "fiemap -v" aligned_btrfs
aligned_btrfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..7]: 26768..26775 8 0x0
1: [8..40967]: 269056..310015 40960 0x2000
2: [40968..40975]: 26776..26783 8 0x1
mnt> /sbin/btrfs fi du unaligned_btrfs aligned_btrfs
Total Exclusive Set shared Filename
20.00MiB 20.00MiB 0.00B unaligned_btrfs
20.01MiB 8.00KiB 20.00MiB aligned_btrfsXFS benchmarks run on same host:
Quoted 3 lines
sudo umount mnt && rm
/tmp/backing.img
truncate --size=2G/tmp/backing.img/sbin/mkfs.xfs/tmp/backing.img
...
= reflink=1 ... data = bsize=4096 blocks=524288, imaxpct=25
...
Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mnt
mnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10 gen_init_cpio -o unaligned_xfs list
...
0.011069 +- 0.000469 seconds time elapsed ( +- 4.24% )
mnt> perf stat -r 10 gen_init_cpio -o aligned_xfs -a 4096 list
...
0.001273 +- 0.000288 seconds time elapsed ( +- 22.60% )
mnt> /sbin/xfs_io -c "fiemap -v" unaligned_xfs
unaligned_xfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..40967]: 106176..147143 40968 0x0
1: [40968..65023]: 147144..171199 24056 0x801
mnt> /sbin/xfs_io -c "fiemap -v" aligned_xfs
aligned_xfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..7]: 120..127 8 0x0
1: [8..40967]: 192..41151 40960 0x2000
2: [40968..40975]: 236728..236735 8 0x0
3: [40976..106495]: 236736..302255 65520 0x801The alignment is best-effort; a stderr message is printed if alignment can't be achieved due to PATH_MAX overrun, with fallback to non-padded filename. This allows it to still be useful for opportunistic alignment, e.g. on aarch64 Btrfs with 64K block-size. Alignment failure messages provide an indicator that reordering of the cpio-manifest may be beneficial.
Archive read performance for reflinked initramfs images may suffer due to the effects of fragmentation, particularly on spinning disks. To mitigate excessive fragmentation, files with lengths less than data_align aren't padded.
--- usr/gen_init_cpio.c | 50 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 12 deletions(-)
usr/gen_init_cpio.c +38 -12
@@ -28,13 +28,15 @@#define CPIO_TRAILER "TRAILER!!!"#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align)) -static char padding[512 ]; +/* zero-padding the filename field for data alignment is limited by PATH_MAX */ +static char padding[PATH_MAX ];static unsigned int offset;static unsigned int ino =721 ;static time_t default_mtime;static bool do_file_mtime;static bool do_csum = false;static int outfd =STDOUT_FILENO ; +static unsigned int dalign;struct file_handler {const char *type;
@@ -359,7 +361,7 @@ static int cpio_mkfile(const char *name, const char *location,int file, retval, len;int rc =-1 ;time_t mtime; -int namesize; +int namesize, namepadlen;unsigned int i;uint32_t csum =0 ;ssize_t this_read;
@@ -407,14 +409,27 @@ static int cpio_mkfile(const char *name, const char *location, } size =0 ; + namepadlen =0 ;for (i =1 ; i <= nlinks; i++) { -/* data goes on last link */ -if (i == nlinks) - size = buf.st_size; -if (name[0 ] =='/' ) name++; namesize =strlen (name) +1 ; + +/* data goes on last link, after any alignment padding */ +if (i == nlinks) + size = buf.st_size; + +if (dalign && size > dalign) { + namepadlen =padlen (offset +CPIO_HDR_LEN + namesize, + dalign); +if (namesize + namepadlen >PATH_MAX ) { +fprintf (stderr, +"%s: best-effort alignment %u missed\n" , + name, dalign); + namepadlen =0 ; + } + } + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08lX%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -429,13 +444,13 @@ static int cpio_mkfile(const char *name, const char *location,1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - namesize,/* namesize */ + namesize + namepadlen,/* namesize */ size ? csum : 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN ||push_buf (name, namesize) <0 - ||push_pad (padlen (offset,4 )) <0 ) + ||push_pad (namepadlen ? namepadlen :padlen (offset,4 )) <0 ) goto error;if (size) {
@@ -552,8 +567,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] [-o <output_path>] <cpio_list>\n" -"\n" +"\t%s [-t <timestamp>] [-c] [-o <output_path>] [-a <data_align>] <cpio_list>\n\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n" "\n"
@@ -590,7 +604,10 @@ static void usage(const char *prog)"The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" "-c: calculate and store 32-bit checksums for file data.\n" -"<output_path>: write cpio to this file instead of stdout\n" , +"<output_path>: write cpio to this file instead of stdout\n" +"<data_align>: attempt to align file data by zero-padding the\n" +"filename field up to data_align. Must be a multiple of 4.\n" +"Alignment is best-effort; PATH_MAX limits filename padding.\n" , prog); }
@@ -632,7 +649,7 @@ int main (int argc, char *argv[]) default_mtime =time (NULL );while (1 ) { -int opt =getopt (argc, argv,"t:cho:" ); +int opt =getopt (argc, argv,"t:cho:a:" );char *invalid;if (opt ==-1 )
@@ -661,6 +678,15 @@ int main (int argc, char *argv[])exit (1 ); }break ; +case 'a' : + dalign =strtoul (optarg, &invalid,10 ); +if (!*optarg || *invalid || (dalign &3 )) { +fprintf (stderr,"Invalid data_align: %s\n" , + optarg); +usage (argv[0 ]); +exit (1 ); + } +break ;case 'h' :case '?' :usage (argv[0 ]);
Signature · 1 line
2.43.0
Nicolas Schier
maintainerRe: [PATCH v2 7/7] gen_init_cpio: add -a <data_align> as reflink optimization
Quoted 92 lines · David Disseldorp
As described in
buffer-format.rst, the existinginitramfs.cextraction logic works fine if the cpio filename field is padded out with trailing zeros, with a caveat that the padded namesize can't exceedPATH_MAX.Add filename zero-padding logic to
gen_init_cpio, which can be triggered via the new -a <data_align> parameter. Performance and storage utilization is improved for Btrfs and XFS workloads, ascopy_file_rangecan reflink the entire source file into a filesystem block-size aligned destination offset within the cpio archive.Btrfs benchmarks run on 6.15.8-1-default (Tumbleweed)
x86_64host:Quoted 2 lines
truncate --size=2G
/tmp/backing.img/sbin/mkfs.btrfs/tmp/backing.img...
Sector size: 4096 (CPU page size: 4096)
...Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mntmnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10gen_init_cpio-ounaligned_btrfslist
...0.023496 +- 0.000472 seconds time elapsed ( +- 2.01% )mnt> perf stat -r 10
gen_init_cpio-oaligned_btrfs-a 4096 list
...0.0010010 +- 0.0000565 seconds time elapsed ( +- 5.65% )mnt>
/sbin/xfs_io-c "fiemap -v"unaligned_btrfsunaligned_btrfs:EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS 0: [0..40967]: 695040..736007 40968 0x1 mnt> /sbin/xfs_io -c "fiemap -v" aligned_btrfs aligned_btrfs: EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS 0: [0..7]: 26768..26775 8 0x0 1: [8..40967]: 269056..310015 40960 0x2000 2: [40968..40975]: 26776..26783 8 0x1 mnt> /sbin/btrfs fi du unaligned_btrfs aligned_btrfs Total Exclusive Set shared Filename 20.00MiB 20.00MiB 0.00B unaligned_btrfs 20.01MiB 8.00KiB 20.00MiB aligned_btrfsXFS benchmarks run on same host:
Quoted 3 lines
sudo umount mnt && rm
/tmp/backing.img
truncate --size=2G/tmp/backing.img/sbin/mkfs.xfs/tmp/backing.img...
= reflink=1 ... data = bsize=4096 blocks=524288, imaxpct=25...
Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mntmnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10gen_init_cpio-ounaligned_xfslist
...0.011069 +- 0.000469 seconds time elapsed ( +- 4.24% )mnt> perf stat -r 10
gen_init_cpio-oaligned_xfs-a 4096 list
...0.001273 +- 0.000288 seconds time elapsed ( +- 22.60% )mnt>
/sbin/xfs_io-c "fiemap -v"unaligned_xfs
unaligned_xfs:EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS 0: [0..40967]: 106176..147143 40968 0x0 1: [40968..65023]: 147144..171199 24056 0x801 mnt> /sbin/xfs_io -c "fiemap -v" aligned_xfs aligned_xfs: EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS 0: [0..7]: 120..127 8 0x0 1: [8..40967]: 192..41151 40960 0x2000 2: [40968..40975]: 236728..236735 8 0x0 3: [40976..106495]: 236736..302255 65520 0x801The alignment is best-effort; a stderr message is printed if alignment can't be achieved due to
PATH_MAXoverrun, with fallback to non-padded filename. This allows it to still be useful for opportunistic alignment, e.g. on aarch64 Btrfs with 64K block-size. Alignment failure messages provide an indicator that reordering of the cpio-manifest may be beneficial.Archive read performance for reflinked initramfs images may suffer due to the effects of fragmentation, particularly on spinning disks. To mitigate excessive fragmentation, files with lengths less than
data_alignaren't padded.
Quoted 4 lines
Signed-off-by David Disseldorp <[email protected]>--- usr/gen_init_cpio.c | 50 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 12 deletions(-)
Thanks! Testing with a massively oversized initramfs (600MB) was fun:
from 2:44 down to 38s.
Questions that pop up in my mind:
Now, how can we make other benefit from this? Might it make sense to introduce a kconfig variable for initramfs alignment -- even though this is just a build-time optimisation of few seconds?
Kind regards,
Nicolas
Re: [PATCH v2 7/7] gen_init_cpio: add -a <data_align> as reflink optimization
Oops, I somehow missed this comment...
On Mon, 18 Aug 2025 21:23:36 +0200, Nicolas Schier wrote:
...
Quoted 2 lines
Thanks! Testing with a massively oversized initramfs (600MB) was fun:
from 2:44 down to 38s.
Thanks for the benchmarking :)
Quoted 4 lines
Questions that pop up in my mind:
Now, how can we make other benefit from this? Might it make sense to introduce a kconfig variable for initramfs alignment -- even though this is just a build-time optimisation of few seconds?
A kconfig option with non-aligned as default isn't a bad idea, but at this stage I don't think it's worth adding. Mostly because reflinks are very FS / arch / environment specific and the extra fragmentation may have unintended consequences.
Thanks, David
Quoted 22 lines · David Disseldorp
This patchset adds
copy_file_range()support togen_init_cpio. When combined with data segment alignment, large-file archiving performance is improved on Btrfs and XFS due to reflinks (see 7/7 benchmarks).cpio data segment alignment is provided by "bending" the newc spec to zero-pad the filename field. GNU cpio and Linux initramfs extractors handle this fine as long as
PATH_MAXisn't exceeded.Changes since v1 RFC
- add alignment patches 6-7
- slightly rework commit and error messages
- renamel->lento avoid 1/i confusionDavid Disseldorp (7):
gen_init_cpio: write to fd instead of stdout stream
gen_init_cpio: support -o <output_path> parameter
gen_init_cpio: attemptcopy_file_rangefor file data
gen_init_cpio: avoid duplicate strlen callsgen_initramfs.sh: usegen_init_cpio-o parameter
docs: initramfs: file data alignment via name padding
gen_init_cpio: add -a <data_align> as reflink optimization
Quoted 6 lines
.../driver-api/early-userspace/buffer-format.rst | 5 + usr/gen_init_cpio.c | 234 ++++++++++++++------- usr/gen_initramfs.sh | 7 +- 3 files changed, 166 insertions(+), 80 deletions(-)
Thanks for the series! I have found only a minor nick pick and some few bike-shedding things.
Kind regards,
Nicolas
Quoted 30 lines · Nicolas Schier
Quoted 22 lines · David Disseldorp
This patchset adds
copy_file_range()support togen_init_cpio. When combined with data segment alignment, large-file archiving performance is improved on Btrfs and XFS due to reflinks (see 7/7 benchmarks).cpio data segment alignment is provided by "bending" the newc spec to zero-pad the filename field. GNU cpio and Linux initramfs extractors handle this fine as long as
PATH_MAXisn't exceeded.Changes since v1 RFC
- add alignment patches 6-7
- slightly rework commit and error messages
- renamel->lento avoid 1/i confusionDavid Disseldorp (7):
gen_init_cpio: write to fd instead of stdout stream
gen_init_cpio: support -o <output_path> parameter
gen_init_cpio: attemptcopy_file_rangefor file data
gen_init_cpio: avoid duplicate strlen callsgen_initramfs.sh: usegen_init_cpio-o parameter
docs: initramfs: file data alignment via name padding
gen_init_cpio: add -a <data_align> as reflink optimizationQuoted 6 lines
.../driver-api/early-userspace/buffer-format.rst | 5 + usr/gen_init_cpio.c | 234 ++++++++++++++------- usr/gen_initramfs.sh | 7 +- 3 files changed, 166 insertions(+), 80 deletions(-)
Quoted 4 lines
Thanks for the series! I have found only a minor nick pick and some few bike-shedding things.
Reviewed-by Nicolas Schier <[email protected]>
Thanks a lot for the review, Nicolas!
I'll post a v3 with your suggestions addressed, as I have one further initramfs_test.c patch to padded filename test coverage.
Cheers, David