gen_init_cpio: add copy_file_range / reflink support
Subsystems (4)
v3 cover gen_init_cpio: add copy_file_range / reflink support
This patchset adds copy_file_range() support to gen_init_cpio. When combined with data segment alignment, large-file archiving performance is improved on Btrfs and XFS due to reflinks (see patch 7 benchmarks).
cpio data segment alignment is provided by "bending" the newc spec to zero-pad the filename field. GNU cpio and Linux initramfs extractors handle this fine as long as PATH_MAX isn't exceeded. A kernel initramfs extraction unit test for this is provided.
Changes since v2
- add Nicolas' Reviewed-by tag to patches 1-7
- add patch 8 test for extracting a cpio archive with filename padding
- use trailing '||' for multi-line if conditions instead of prefix
- refer to -o output_file in usage instead of output_path
- define _GNU_SOURCE alongside O_LARGEFILE use, instead of later
Changes since v1 RFC
- add alignment patches 6-7
- slightly rework commit and error messages
- rename l->len to avoid 1/i confusion
PATCH v3 1/8 gen_init_cpio: write to fd instead of stdout stream
In preparation for more efficient archiving using copy_file_range(), switch from writing archive data to stdout to using STDOUT_FILENO and I/O via write(), dprintf(), etc.
Basic I/O error handling is added to cover cases such as ENOSPC. Partial writes are treated as errors.
--- usr/gen_init_cpio.c | 139 ++++++++++++++++++++++++++------------------ 1 file changed, 81 insertions(+), 58 deletions(-)
usr/gen_init_cpio.c +81 -58
@@ -23,64 +23,71 @@#define xstr (s) #s#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b)) +#define CPIO_HDR_LEN 110 +#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align)) +static char padding[512 ];static unsigned int offset;static unsigned int ino =721 ;static time_t default_mtime;static bool do_file_mtime;static bool do_csum = false; +static int outfd =STDOUT_FILENO ;struct file_handler {const char *type;int (*handler)(const char *line); }; -static void push_string (const char *name) +static int push_string (const char *name) {unsigned int name_len =strlen (name) +1 ; +ssize_t len; + + len =write (outfd, name, name_len); +if (len != name_len) +return -1 ; -fputs (name, stdout); -putchar (0 ); offset += name_len; +return 0 ; } -static void push_pad (void ) +static int push_pad (size_t padlen) { -while (offset &3 ) { -putchar (0 ); - offset++; - } +ssize_t len =0 ; + +if (!padlen) +return 0 ; + +if (padlen <sizeof (padding)) + len =write (outfd, padding, padlen); +if (len != padlen) +return -1 ; + + offset += padlen; +return 0 ; } -static void push_rest (const char *name) +static int push_rest (const char *name) {unsigned int name_len =strlen (name) +1 ; -unsigned int tmp_ofs; +ssize_t len; -fputs (name, stdout); -putchar (0 ); - offset += name_len; + len =write (outfd, name, name_len); +if (len != name_len) +return -1 ; - tmp_ofs = name_len +110 ; -while (tmp_ofs &3 ) { -putchar (0 ); - offset++; - tmp_ofs++; - } -} + offset += name_len; -static void push_hdr (const char *s) -{ -fputs (s, stdout); - offset +=110 ; +return push_pad (padlen (name_len +CPIO_HDR_LEN ,4 )); } -static void cpio_trailer (void ) +static int cpio_trailer (void ) { -char s[256 ];const char name[] ="TRAILER!!!" ; +int len; - sprintf(s, "%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ 0, /* ino */
@@ -96,23 +103,24 @@ static void cpio_trailer(void)0 ,/* rminor */ (unsigned )strlen (name)+1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; -while (offset %512 ) { -putchar (0 ); - offset++; - } +if (len !=CPIO_HDR_LEN || +push_rest (name) <0 || +push_pad (padlen (offset,512 )) <0 ) +return -1 ; + +return 0 ; }static int cpio_mkslink (const char *name,const char *target,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int len;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -128,12 +136,17 @@ static int cpio_mkslink(const char *name, const char *target,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); -push_string (target); -push_pad (); + offset += len; + +if (len !=CPIO_HDR_LEN || +push_string (name) <0 || +push_pad (padlen (offset,4 )) <0 || +push_string (target) <0 || +push_pad (padlen (offset,4 )) <0 ) +return -1 ; +return 0 ; + }static int cpio_mkslink_line (const char *line)
@@ -157,11 +170,11 @@ static int cpio_mkslink_line(const char *line)static int cpio_mkgeneric (const char *name,unsigned int mode,uid_t uid,gid_t gid) { -char s[256 ]; +int len;if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -177,8 +190,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,0 ,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; + +if (len !=CPIO_HDR_LEN || +push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -246,7 +263,7 @@ static int cpio_mknod(const char *name, unsigned int mode, uid_t uid, gid_t gid, char dev_type, unsigned int maj, unsigned int min) { - char s[256]; + int len;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -255,7 +272,7 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino++, /* ino */
@@ -271,8 +288,12 @@ static int cpio_mknod(const char *name, unsigned int mode, min,/* rminor */ (unsigned )strlen (name) +1 ,/* namesize */ 0); /* chksum */ -push_hdr (s); -push_rest (name); + offset += len; + +if (len !=CPIO_HDR_LEN || +push_rest (name) <0 ) +return -1 ; +return 0 ; }
@@ -324,11 +345,9 @@ static int cpio_mkfile(const char *name, const char *location, unsigned int mode, uid_t uid, gid_t gid, unsigned int nlinks) { - char s[256]; struct stat buf;unsigned long size; -int file; -int retval; +int file, retval, len;int rc =-1 ;time_t mtime;int namesize;
@@ -386,7 +405,7 @@ static int cpio_mkfile(const char *name, const char *location,if (name[0 ] =='/' ) name++; namesize =strlen (name) +1 ; - sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX" + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08lX%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */ ino, /* ino */
@@ -402,9 +421,12 @@ static int cpio_mkfile(const char *name, const char *location,0 ,/* rminor */ namesize,/* namesize */ size ? csum : 0); /* chksum */ -push_hdr (s); -push_string (name); -push_pad (); + offset += len; + +if (len !=CPIO_HDR_LEN || +push_string (name) <0 || +push_pad (padlen (offset,4 )) <0 ) + goto error;while (size) {unsigned char filebuf[65536 ];
@@ -417,14 +439,15 @@ static int cpio_mkfile(const char *name, const char *location, goto error; } -if (fwrite (filebuf, this_read,1 , stdout) !=1 ) { +if (write (outfd, filebuf, this_read) != this_read) {fprintf (stderr,"writing filebuf failed\n" ); goto error; } offset += this_read; size -= this_read; } -push_pad (); +if (push_pad (padlen (offset,4 )) <0 ) + goto error; name += namesize; }
@@ -691,7 +714,7 @@ int main (int argc, char *argv[]) } }if (ec ==0 ) -cpio_trailer (); + ec =cpio_trailer ();exit (ec); }
Signature · 1 line
2.43.0
PATCH v3 2/8 gen_init_cpio: support -o <output_file> parameter
In reply to David DisseldorpThis is another preparatory change to allow for reflink-optimized cpio archives with file data written / cloned via copy_file_range(). The output file is truncated prior to write, so that it maps to usr/gen_initramfs.sh usage. It may make sense to offer an append option in future, for easier archive concatenation.
--- usr/gen_init_cpio.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-)
usr/gen_init_cpio.c +16 -4
@@ -1,4 +1,5 @@// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE#include <stdio.h> #include <stdlib.h> #include <stdint.h>
@@ -110,7 +111,7 @@ static int cpio_trailer(void) push_pad(padlen(offset, 512)) < 0)return -1 ; -return 0 ; +return fsync (outfd); } static int cpio_mkslink(const char *name, const char *target,
@@ -532,7 +533,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] <cpio_list>\n" +"\t%s [-t <timestamp>] [-c] [-o <output_file>] <cpio_list>\n" "\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n"
@@ -569,7 +570,8 @@ static void usage(const char *prog)"as mtime for symlinks, directories, regular and special files.\n" "The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" -"-c: calculate and store 32-bit checksums for file data.\n" , +"-c: calculate and store 32-bit checksums for file data.\n" +"<output_file>: write cpio to this file instead of stdout\n" , prog); }
@@ -611,7 +613,7 @@ int main (int argc, char *argv[]) default_mtime =time (NULL );while (1 ) { -int opt =getopt (argc, argv,"t:ch" ); +int opt =getopt (argc, argv,"t:cho:" );char *invalid;if (opt ==-1 )
@@ -630,6 +632,16 @@ int main (int argc, char *argv[])case 'c' : do_csum = true;break ; +case 'o' : + outfd =open (optarg, +O_WRONLY |O_CREAT |O_LARGEFILE |O_TRUNC , +0600 ); +if (outfd <0 ) { +fprintf (stderr,"failed to open %s\n" , optarg); +usage (argv[0 ]); +exit (1 ); + } +break ;case 'h' :case '?' :usage (argv[0 ]);
Signature · 1 line
2.43.0
PATCH v3 3/8 gen_init_cpio: attempt copy_file_range for file data
In reply to David DisseldorpThe copy_file_range syscall can improve copy performance by cloning extents between cpio archive source and destination files. Existing read / write based copy logic is retained for fallback in case the copy_file_range syscall is unsupported or unavailable due to cross-filesystem EXDEV, etc.
Clone or reflink, as opposed to copy, of source file extents into the output cpio archive may (e.g. on Btrfs and XFS) require alignment of the output to the filesystem block size. This could be achieved by inserting padding entries into the cpio archive manifest.
--- usr/gen_init_cpio.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-)
usr/gen_init_cpio.c +12 -1
@@ -354,6 +354,7 @@ static int cpio_mkfile(const char *name, const char *location,int namesize;unsigned int i;uint32_t csum =0 ; +ssize_t this_read; mode |=S_IFREG ;
@@ -429,9 +430,19 @@ static int cpio_mkfile(const char *name, const char *location, push_pad(padlen(offset, 4)) < 0) goto error; +if (size) { + this_read =copy_file_range (file,NULL , outfd,NULL , size,0 ); +if (this_read >0 ) { +if (this_read > size) + goto error; + offset += this_read; + size -= this_read; + } +/* short or failed copy falls back to read/write... */ + } +while (size) {unsigned char filebuf[65536 ]; -ssize_t this_read;size_t this_size =MIN (size,sizeof (filebuf)); this_read =read (file, filebuf, this_size);
Signature · 1 line
2.43.0
PATCH v3 4/8 gen_init_cpio: avoid duplicate strlen calls
In reply to David DisseldorpWe determine the filename length for the cpio header, so shouldn't recalculate it when writing out the filename.
--- usr/gen_init_cpio.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-)
usr/gen_init_cpio.c +24 -16
@@ -25,6 +25,7 @@#define str (s) xstr(s)#define MIN (a, b) ((a) < (b) ? (a) : (b))#define CPIO_HDR_LEN 110 +#define CPIO_TRAILER "TRAILER!!!"#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align))static char padding[512 ];
@@ -40,9 +41,8 @@ struct file_handler {int (*handler)(const char *line); }; -static int push_string (const char *name) +static int push_buf (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t len; len =write (outfd, name, name_len);
@@ -69,9 +69,8 @@ static int push_pad(size_t padlen)return 0 ; } -static int push_rest (const char *name) +static int push_rest (const char *name,size_t name_len) { -unsigned int name_len =strlen (name) +1 ;ssize_t len; len =write (outfd, name, name_len);
@@ -85,8 +84,8 @@ static int push_rest(const char *name)static int cpio_trailer (void ) { -const char name[] ="TRAILER!!!" ;int len; +unsigned int namesize =sizeof (CPIO_TRAILER ); len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X",
@@ -102,12 +101,12 @@ static int cpio_trailer(void)0 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name)+1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN || -push_rest (name) <0 || +push_rest (CPIO_TRAILER , namesize) <0 ||push_pad (padlen (offset,512 )) <0 )return -1 ;
@@ -118,9 +117,12 @@ static int cpio_mkslink(const char *name, const char *target, unsigned int mode, uid_t uid, gid_t gid) { int len; +unsigned int namesize, targetsize =strlen (target) +1 ;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -130,19 +132,19 @@ static int cpio_mkslink(const char *name, const char *target, (long ) gid,/* gid */ 1 ,/* nlink */ (long ) default_mtime,/* mtime */ - (unsigned )strlen (target)+1 ,/* filesize */ + targetsize,/* filesize */ 3 ,/* major */ 1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN || -push_string (name) <0 || +push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 || -push_string (target) <0 || +push_buf (target, targetsize) <0 ||push_pad (padlen (offset,4 )) <0 )return -1 ;
@@ -172,9 +174,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode, uid_t uid, gid_t gid) { int len; +unsigned int namesize;if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -189,12 +194,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN || -push_rest (name) <0 ) +push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -265,6 +270,7 @@ static int cpio_mknod(const char *name, unsigned int mode, unsigned int maj, unsigned int min) { int len; +unsigned int namesize;if (dev_type =='b' ) mode |=S_IFBLK ;
@@ -273,6 +279,8 @@ static int cpio_mknod(const char *name, unsigned int mode,if (name[0 ] =='/' ) name++; + namesize =strlen (name) +1 ; + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08X%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -287,12 +295,12 @@ static int cpio_mknod(const char *name, unsigned int mode,1 ,/* minor */ maj,/* rmajor */ min,/* rminor */ - (unsigned )strlen (name) +1 ,/* namesize */ + namesize,/* namesize */ 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN || -push_rest (name) <0 ) +push_rest (name, namesize) <0 )return -1 ;return 0 ;
@@ -426,7 +434,7 @@ static int cpio_mkfile(const char *name, const char *location, offset += len;if (len !=CPIO_HDR_LEN || -push_string (name) <0 || +push_buf (name, namesize) <0 ||push_pad (padlen (offset,4 )) <0 ) goto error;
Signature · 1 line
2.43.0
PATCH v3 5/8 gen_initramfs.sh: use gen_init_cpio -o parameter
In reply to David Disseldorpgen_init_cpio can now write to a file directly, so use it when gen_initramfs.sh is called with -o (e.g. usr/Makefile invocation).
--- usr/gen_initramfs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
usr/gen_initramfs.sh +4 -3
@@ -193,7 +193,8 @@ root_gid=0 dep_list= timestamp= cpio_list=$(mktemp ${TMPDIR:-/tmp}/cpiolist.XXXXXX) -output="/dev/stdout" +# gen_init_cpio writes to stdout by default +output="" trap "rm -f $cpio_list" EXIT
@@ -207,7 +208,7 @@ while [ $# -gt 0 ]; do shift ;; "-o") # generate cpio image named $1 - output="$1" + output="-o $1" shift ;; "-u") # map $1 to uid=0 (root)
@@ -246,4 +247,4 @@ done # If output_file is set we will generate cpio archive # we are careful to delete tmp files -usr/gen_init_cpio $timestamp $cpio_list > $output +usr/gen_init_cpio $output $timestamp $cpio_list
Signature · 1 line
2.43.0
PATCH v3 6/8 docs: initramfs: file data alignment via name padding
In reply to David DisseldorpThe existing cpio extraction logic reads (maximum PATH_MAX) name_len bytes from the archive into the collected name buffer and ensures that the trailing byte is a null-terminator. This allows the actual file name to be shorter than name_len, with the name string simply zero-terminated prior to the last byte.
Initramfs generators, such as dracut-cpio[1], can take advantage of name zero-padding to align file data segments within the archive to filesystem block boundaries. Block boundary alignment may allow the copy_file_range syscall to reflink archive source and destination extents.
--- Documentation/driver-api/early-userspace/buffer-format.rst | 5 +++++ 1 file changed, 5 insertions(+)
Documentation/driver-api/early-userspace/buffer-format.rst +5 -0
@@ -86,6 +86,11 @@ c_mtime is ignored unless CONFIG_INITRAMFS_PRESERVE_MTIME=y is set. The c_filesize should be zero for any file which is not a regular file or symlink. +c_namesize may account for more than one trailing '\0', as long as the +value doesn't exceed PATH_MAX. This can be useful for ensuring that a +subsequent file data segment is aligned, e.g. to a filesystem block +boundary. + The c_chksum field contains a simple 32-bit unsigned sum of all the bytes in the data field. cpio(1) refers to this as "crc", which is clearly incorrect (a cyclic redundancy check is a different and
Signature · 1 line
2.43.0
PATCH v3 7/8 gen_init_cpio: add -a <data_align> as reflink optimization
In reply to David DisseldorpAs described in buffer-format.rst, the existing initramfs.c extraction logic works fine if the cpio filename field is padded out with trailing zeros, with a caveat that the padded namesize can't exceed PATH_MAX.
Add filename zero-padding logic to gen_init_cpio, which can be triggered via the new -a <data_align> parameter. Performance and storage utilization is improved for Btrfs and XFS workloads, as copy_file_range can reflink the entire source file into a filesystem block-size aligned destination offset within the cpio archive.
Btrfs benchmarks run on 6.15.8-1-default (Tumbleweed) x86_64 host:
Quoted 2 lines
truncate --size=2G
/tmp/backing.img/sbin/mkfs.btrfs/tmp/backing.img
...
Sector size: 4096 (CPU page size: 4096)
...
Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mnt
mnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10 gen_init_cpio -o unaligned_btrfs list
...
0.023496 +- 0.000472 seconds time elapsed ( +- 2.01% )
mnt> perf stat -r 10 gen_init_cpio -o aligned_btrfs -a 4096 list
...
0.0010010 +- 0.0000565 seconds time elapsed ( +- 5.65% )
mnt> /sbin/xfs_io -c "fiemap -v" unaligned_btrfs unaligned_btrfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..40967]: 695040..736007 40968 0x1
mnt> /sbin/xfs_io -c "fiemap -v" aligned_btrfs
aligned_btrfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..7]: 26768..26775 8 0x0
1: [8..40967]: 269056..310015 40960 0x2000
2: [40968..40975]: 26776..26783 8 0x1
mnt> /sbin/btrfs fi du unaligned_btrfs aligned_btrfs
Total Exclusive Set shared Filename
20.00MiB 20.00MiB 0.00B unaligned_btrfs
20.01MiB 8.00KiB 20.00MiB aligned_btrfsXFS benchmarks run on same host:
Quoted 3 lines
sudo umount mnt && rm
/tmp/backing.img
truncate --size=2G/tmp/backing.img/sbin/mkfs.xfs/tmp/backing.img
...
= reflink=1 ... data = bsize=4096 blocks=524288, imaxpct=25
...
Quoted 3 lines
sudo mount
/tmp/backing.imgmnt
sudo chown $USER mnt
cd mnt
mnt> dd if=/dev/urandom of=foo bs=1M count=20 && cat foo >/dev/null
...
mnt> echo "file /foo foo 0755 0 0" > list
mnt> perf stat -r 10 gen_init_cpio -o unaligned_xfs list
...
0.011069 +- 0.000469 seconds time elapsed ( +- 4.24% )
mnt> perf stat -r 10 gen_init_cpio -o aligned_xfs -a 4096 list
...
0.001273 +- 0.000288 seconds time elapsed ( +- 22.60% )
mnt> /sbin/xfs_io -c "fiemap -v" unaligned_xfs
unaligned_xfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..40967]: 106176..147143 40968 0x0
1: [40968..65023]: 147144..171199 24056 0x801
mnt> /sbin/xfs_io -c "fiemap -v" aligned_xfs
aligned_xfs:
EXT: FILE-OFFSET BLOCK-RANGE TOTAL FLAGS
0: [0..7]: 120..127 8 0x0
1: [8..40967]: 192..41151 40960 0x2000
2: [40968..40975]: 236728..236735 8 0x0
3: [40976..106495]: 236736..302255 65520 0x801The alignment is best-effort; a stderr message is printed if alignment can't be achieved due to PATH_MAX overrun, with fallback to non-padded filename. This allows it to still be useful for opportunistic alignment, e.g. on aarch64 Btrfs with 64K block-size. Alignment failure messages provide an indicator that reordering of the cpio-manifest may be beneficial.
Archive read performance for reflinked initramfs images may suffer due to the effects of fragmentation, particularly on spinning disks. To mitigate excessive fragmentation, files with lengths less than data_align aren't padded.
--- usr/gen_init_cpio.c | 49 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 11 deletions(-)
usr/gen_init_cpio.c +38 -11
@@ -28,13 +28,15 @@#define CPIO_TRAILER "TRAILER!!!"#define padlen (_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align)) -static char padding[512 ]; +/* zero-padding the filename field for data alignment is limited by PATH_MAX */ +static char padding[PATH_MAX ];static unsigned int offset;static unsigned int ino =721 ;static time_t default_mtime;static bool do_file_mtime;static bool do_csum = false;static int outfd =STDOUT_FILENO ; +static unsigned int dalign;struct file_handler {const char *type;
@@ -359,7 +361,7 @@ static int cpio_mkfile(const char *name, const char *location,int file, retval, len;int rc =-1 ;time_t mtime; -int namesize; +int namesize, namepadlen;unsigned int i;uint32_t csum =0 ;ssize_t this_read;
@@ -407,14 +409,27 @@ static int cpio_mkfile(const char *name, const char *location, } size =0 ; + namepadlen =0 ;for (i =1 ; i <= nlinks; i++) { -/* data goes on last link */ -if (i == nlinks) - size = buf.st_size; -if (name[0 ] =='/' ) name++; namesize =strlen (name) +1 ; + +/* data goes on last link, after any alignment padding */ +if (i == nlinks) + size = buf.st_size; + +if (dalign && size > dalign) { + namepadlen =padlen (offset +CPIO_HDR_LEN + namesize, + dalign); +if (namesize + namepadlen >PATH_MAX ) { +fprintf (stderr, +"%s: best-effort alignment %u missed\n" , + name, dalign); + namepadlen =0 ; + } + } + len = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX" "%08lX%08X%08X%08X%08X%08X%08X", do_csum ? "070702" : "070701", /* magic */
@@ -429,13 +444,13 @@ static int cpio_mkfile(const char *name, const char *location,1 ,/* minor */ 0 ,/* rmajor */ 0 ,/* rminor */ - namesize,/* namesize */ + namesize + namepadlen,/* namesize */ size ? csum : 0); /* chksum */ offset += len;if (len !=CPIO_HDR_LEN ||push_buf (name, namesize) <0 || -push_pad (padlen (offset,4 )) <0 ) +push_pad (namepadlen ? namepadlen :padlen (offset,4 )) <0 ) goto error;if (size) {
@@ -552,7 +567,7 @@ static int cpio_mkfile_line(const char *line)static void usage (const char *prog) {fprintf (stderr,"Usage:\n" -"\t%s [-t <timestamp>] [-c] [-o <output_file>] <cpio_list>\n" +"\t%s [-t <timestamp>] [-c] [-o <output_file>] [-a <data_align>] <cpio_list>\n" "\n" "<cpio_list> is a file containing newline separated entries that\n" "describe the files to be included in the initramfs archive:\n"
@@ -590,7 +605,10 @@ static void usage(const char *prog)"The default is to use the current time for all files, but\n" "preserve modification time for regular files.\n" "-c: calculate and store 32-bit checksums for file data.\n" -"<output_file>: write cpio to this file instead of stdout\n" , +"<output_file>: write cpio to this file instead of stdout\n" +"<data_align>: attempt to align file data by zero-padding the\n" +"filename field up to data_align. Must be a multiple of 4.\n" +"Alignment is best-effort; PATH_MAX limits filename padding.\n" , prog); }
@@ -632,7 +650,7 @@ int main (int argc, char *argv[]) default_mtime =time (NULL );while (1 ) { -int opt =getopt (argc, argv,"t:cho:" ); +int opt =getopt (argc, argv,"t:cho:a:" );char *invalid;if (opt ==-1 )
@@ -661,6 +679,15 @@ int main (int argc, char *argv[])exit (1 ); }break ; +case 'a' : + dalign =strtoul (optarg, &invalid,10 ); +if (!*optarg || *invalid || (dalign &3 )) { +fprintf (stderr,"Invalid data_align: %s\n" , + optarg); +usage (argv[0 ]); +exit (1 ); + } +break ;case 'h' :case '?' :usage (argv[0 ]);
Signature · 1 line
2.43.0
PATCH v3 8/8 initramfs_test: add filename padding test case
In reply to David DisseldorpConfirm that cpio filenames with multiple trailing zeros (accounted for in namesize) extract successfully.
--- init/initramfs_test.c | 68 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-)
init/initramfs_test.c +67 -1
@@ -45,8 +45,11 @@ static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, char *out) c->mtime, c->filesize, c->devmajor, c->devminor, c->rdevmajor, c->rdevminor, c->namesize, c->csum, c->fname) + 1; +pr_debug ("packing (%zu): %.*s\n" , thislen, (int )thislen, pos); - off += thislen; +if (thislen !=CPIO_HDRLEN + c->namesize) +pr_debug ("padded to: %u\n" ,CPIO_HDRLEN + c->namesize); + off +=CPIO_HDRLEN + c->namesize;while (off &3 ) out[off++] ='\0' ;
@@ -383,6 +386,68 @@ static void __init initramfs_test_many(struct kunit *test)kfree (cpio_srcbuf); } +/* +* An initramfs filename is namesize in length, including the zero-terminator. +* A filename can be zero-terminated prior to namesize, with the remainder used +* as padding. This can be useful for e.g. alignment of file data segments with +* a 4KB filesystem block, allowing for extent sharing (reflinks) between cpio +* source and destination. This hack works with both GNU cpio and initramfs, as +* long as PATH_MAX isn't exceeded. +*/ +static void __init initramfs_test_fname_pad (struct kunit *test) +{ +char *err; +size_t len; +struct file *file; +char fdata[] ="this file data is aligned at 4K in the archive" ; +struct test_fname_pad { +char padded_fname[4096 -CPIO_HDRLEN ]; +char cpio_srcbuf[CPIO_HDRLEN +PATH_MAX +3 +sizeof (fdata)]; + } *tbufs =kzalloc (sizeof (struct test_fname_pad ),GFP_KERNEL ); +struct initramfs_test_cpio c[] = { { + .magic ="070701" , + .ino =1 , + .mode =S_IFREG |0777 , + .uid =0 , + .gid =0 , + .nlink =1 , + .mtime =1 , + .filesize =0 , + .devmajor =0 , + .devminor =1 , + .rdevmajor =0 , + .rdevminor =0 , +/* align file data at 4K archive offset via padded fname */ + .namesize =4096 -CPIO_HDRLEN , + .csum =0 , + .fname = tbufs->padded_fname, + .data = fdata, + .filesize =sizeof (fdata), + } }; + +memcpy (tbufs->padded_fname,"padded_fname" ,sizeof ("padded_fname" )); + len =fill_cpio (c,ARRAY_SIZE (c), tbufs->cpio_srcbuf); + + err =unpack_to_rootfs (tbufs->cpio_srcbuf, len); +KUNIT_EXPECT_NULL (test, err); + + file =filp_open (c[0 ].fname,O_RDONLY ,0 ); +if (IS_ERR (file)) { +KUNIT_FAIL (test,"open failed" ); + goto out; + } + +/* read back file contents into @cpio_srcbuf and confirm match */ + len =kernel_read (file, tbufs->cpio_srcbuf, c[0 ].filesize,NULL ); +KUNIT_EXPECT_EQ (test, len, c[0 ].filesize); +KUNIT_EXPECT_MEMEQ (test, tbufs->cpio_srcbuf, c[0 ].data, len); + +fput (file); +KUNIT_EXPECT_EQ (test,init_unlink (c[0 ].fname),0 ); +out: +kfree (tbufs); +} +/* * The kunit_case/_suite struct cannot be marked as __initdata as this will be * used in debugfs to retrieve results after test has run.
@@ -394,6 +459,7 @@ static struct kunit_case __refdata initramfs_test_cases[] = {KUNIT_CASE (initramfs_test_csum),KUNIT_CASE (initramfs_test_hardlink),KUNIT_CASE (initramfs_test_many), + KUNIT_CASE(initramfs_test_fname_pad), {}, };
Signature · 1 line
2.43.0
kernel test robot
build botRe: [PATCH v3 8/8] initramfs_test: add filename padding test case
kernel test robot noticed the following build warnings:
Hi David,
kernel test robot noticed the following build warnings:
[auto build test WARNING on brauner-vfs/vfs.all]
[also build test WARNING on linus/master v6.17-rc2 next-20250819] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/David-Disseldorp/gen_init_cpio-write-to-fd-instead-of-stdout-stream/20250819-115406 base: https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs.all patch link: https://lore.kernel.org/r/20250819032607.28727-9-ddiss%40suse.de patch subject: [PATCH v3 8/8] initramfs_test: add filename padding test case config: sparc64-randconfig-r121-20250819 (https://download.01.org/0day-ci/archive/20250820/[email protected]/config) compiler: clang version 22.0.0git (https://github.com/llvm/llvm-project 93d24b6b7b148c47a2fa228a4ef31524fa1d9f3f) reproduce: (https://download.01.org/0day-ci/archive/20250820/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
sparse warnings: (new ones prefixed by >>)
>> init/initramfs_test.c:415:18: sparse: sparse: Initializer entry defined twice
init/initramfs_test.c:425:18: sparse: also defined here
vim +415 init/initramfs_test.c
388
389 /*
390 * An initramfs filename is namesize in length, including the zero-terminator.
391 * A filename can be zero-terminated prior to namesize, with the remainder used
392 * as padding. This can be useful for e.g. alignment of file data segments with
393 * a 4KB filesystem block, allowing for extent sharing (reflinks) between cpio
394 * source and destination. This hack works with both GNU cpio and initramfs, as
395 * long as PATH_MAX isn't exceeded.
396 */
397 static void __init initramfs_test_fname_pad(struct kunit *test)
398 {
399 char *err;
400 size_t len;
401 struct file *file;
402 char fdata[] = "this file data is aligned at 4K in the archive";
403 struct test_fname_pad {
404 char padded_fname[4096 - CPIO_HDRLEN];
405 char cpio_srcbuf[CPIO_HDRLEN + PATH_MAX + 3 + sizeof(fdata)];
406 } *tbufs = kzalloc(sizeof(struct test_fname_pad), GFP_KERNEL);
407 struct initramfs_test_cpio c[] = { {
408 .magic = "070701",
409 .ino = 1,
410 .mode = S_IFREG | 0777,
411 .uid = 0,
412 .gid = 0,
413 .nlink = 1,
414 .mtime = 1,
> 415 .filesize = 0,
416 .devmajor = 0,
417 .devminor = 1,
418 .rdevmajor = 0,
419 .rdevminor = 0,
420 /* align file data at 4K archive offset via padded fname */
421 .namesize = 4096 - CPIO_HDRLEN,
422 .csum = 0,
423 .fname = tbufs->padded_fname,
424 .data = fdata,
425 .filesize = sizeof(fdata),
426 } };
427
428 memcpy(tbufs->padded_fname, "padded_fname", sizeof("padded_fname"));
429 len = fill_cpio(c, ARRAY_SIZE(c), tbufs->cpio_srcbuf);
430
431 err = unpack_to_rootfs(tbufs->cpio_srcbuf, len);
432 KUNIT_EXPECT_NULL(test, err);
433
434 file = filp_open(c[0].fname, O_RDONLY, 0);
435 if (IS_ERR(file)) {
436 KUNIT_FAIL(test, "open failed");
437 goto out;
438 }
439
440 /* read back file contents into @cpio_srcbuf and confirm match */
441 len = kernel_read(file, tbufs->cpio_srcbuf, c[0].filesize, NULL);
442 KUNIT_EXPECT_EQ(test, len, c[0].filesize);
443 KUNIT_EXPECT_MEMEQ(test, tbufs->cpio_srcbuf, c[0].data, len);
444
445 fput(file);
446 KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
447 out:
448 kfree(tbufs);
449 }
450
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Re: [PATCH v3 8/8] initramfs_test: add filename padding test case
Quoted 3 lines · kernel test robot
sparse warnings: (new ones prefixed by >>)
Quoted 1 line
Quoted 1 line
init/initramfs_test.c:415:18: sparse: sparse: Initializer entry defined twice
init/initramfs_test.c:425:18: sparse: also defined here
...
Quoted 9 lines
407 struct initramfs_test_cpio c[] = { { 408 .magic = "070701", 409 .ino = 1, 410 .mode = S_IFREG | 0777, 411 .uid = 0, 412 .gid = 0, 413 .nlink = 1, 414 .mtime = 1,Quoted 1 line
415 .filesize = 0,
...
Quoted 2 lines
425 .filesize = sizeof(fdata), 426 } };
Thanks. I can send a v4 patchset to address this, or otherwise happy to have line 415 removed by a maintainer when merged.
Re: [PATCH v3 8/8] initramfs_test: add filename padding test case
Quoted 15 lines · David Disseldorp
Quoted 3 lines · kernel test robot
sparse warnings: (new ones prefixed by >>)
Quoted 1 line
Quoted 1 line
init/initramfs_test.c:415:18: sparse: sparse: Initializer entry defined twice
init/initramfs_test.c:425:18: sparse: also defined here...
Quoted 9 lines
407 struct initramfs_test_cpio c[] = { { 408 .magic = "070701", 409 .ino = 1, 410 .mode = S_IFREG | 0777, 411 .uid = 0, 412 .gid = 0, 413 .nlink = 1, 414 .mtime = 1,Quoted 1 line
415 .filesize = 0,
Quoted 6 lines
...
Quoted 2 lines
425 .filesize = sizeof(fdata), 426 } };Thanks. I can send a v4 patchset to address this, or otherwise happy to have line 415 removed by a maintainer when merged.
With that change:
Thanks and kind regards
Nicolas
Re: [PATCH v3 8/8] initramfs_test: add filename padding test case
Quoted 11 lines · Nicolas Schier
Quoted 7 lines
Quoted 1 line
Quoted 1 line
415 .filesize = 0,
...
Quoted 2 lines
425 .filesize = sizeof(fdata), 426 } };Thanks. I can send a v4 patchset to address this, or otherwise happy to have line 415 removed by a maintainer when merged.
With that change:
Acked-by Nicolas Schier <[email protected]>
Thanks Nicolas!
Do you have any suggestions regarding how this patchset should proceed - would git://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git
kbuild-next be suitable as a pre-merge-window staging area?
Cheers, David
Nicolas Schier
maintainerRe: [PATCH v3 8/8] initramfs_test: add filename padding test case
Quoted 10 lines · David Disseldorp
Quoted 8 lines · Nicolas Schier
Quoted 7 lines
Quoted 1 line
Quoted 1 line
415 .filesize = 0,
...
Quoted 2 lines
425 .filesize = sizeof(fdata), 426 } };Thanks. I can send a v4 patchset to address this, or otherwise happy to have line 415 removed by a maintainer when merged.
Quoted 8 lines
Quoted 3 lines
With that change:
Acked-by Nicolas Schier <[email protected]>Thanks Nicolas!
Do you have any suggestions regarding how this patchset should proceed - wouldgit://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git
kbuild-next be suitable as a pre-merge-window staging area?
yes, Nathan agreed to pick it up for kbuild-next.
Kind regards,
Nicolas
Quoted 10 lines · David Disseldorp
This patchset adds
copy_file_range()support togen_init_cpio. When combined with data segment alignment, large-file archiving performance is improved on Btrfs and XFS due to reflinks (see patch 7 benchmarks).cpio data segment alignment is provided by "bending" the newc spec to zero-pad the filename field. GNU cpio and Linux initramfs extractors handle this fine as long as
PATH_MAXisn't exceeded. A kernel initramfs extraction unit test for this is provided.[...]
Applied, thanks!
[1/8] gen_init_cpio: write to fd instead of stdout stream
https://git.kernel.org/kbuild/c/1400227773201 [2/8] gen_init_cpio: support -o <output_file> parameter https://git.kernel.org/kbuild/c/ae18b94099b04 [3/8] gen_init_cpio: attempt copy_file_range for file data https://git.kernel.org/kbuild/c/97169cd6d95b3 [4/8] gen_init_cpio: avoid duplicate strlen calls https://git.kernel.org/kbuild/c/348ff9e3c1cf1 [5/8] gen_initramfs.sh: use gen_init_cpio -o parameter https://git.kernel.org/kbuild/c/9135564db4904 [6/8] docs: initramfs: file data alignment via name padding https://git.kernel.org/kbuild/c/7c1f14f6e8e7f [7/8] gen_init_cpio: add -a <data_align> as reflink optimization https://git.kernel.org/kbuild/c/5467e85508fd1 [8/8] initramfs_test: add filename padding test case https://git.kernel.org/kbuild/c/6da752f55bc48
Best regards,
Signature · 1 line
Nathan Chancellor <[email protected]>