Skip to contentSkip to conversation

gen_init_cpio: add copy_file_range support

RFC PATCH Superseded David Disseldorp 1 participant 6 messages Aug 3, 2025 Kbuild

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover gen_init_cpio: add copy_file_range support

This patchset adds copy_file_range() support to gen_init_cpio.

I've flagged it as RFC for a few reasons:
- more testing is needed
- it might not be useful for others
+ see message regarding reflink alignment requirements in PATCH 3/5
- I should probably provide some benchmark results to justify

David Disseldorp (5):
gen_init_cpio: write to fd instead of stdout stream
gen_init_cpio: support -o <output_path> parameter
gen_init_cpio: attempt copy_file_range for file data
gen_init_cpio: avoid duplicate strlen calls gen_initramfs.sh: use gen_init_cpio -o parameter

 usr/gen_init_cpio.c  | 192 +++++++++++++++++++++++++++----------------
 usr/gen_initramfs.sh |   7 +-
 2 files changed, 126 insertions(+), 73 deletions(-)
#1 lore raw

RFC PATCH 1/5 gen_init_cpio: write to fd instead of stdout stream

In preparation for more efficient archiving using copy_file_range(), switch from writing archive data to stdout to using STDOUT_FILENO and I/O via write(), dprintf(), etc.
Basic I/O error handling is added to cover cases such as ENOSPC. Partial writes are treated as errors.

---
 usr/gen_init_cpio.c | 139 ++++++++++++++++++++++++++------------------
 1 file changed, 81 insertions(+), 58 deletions(-)
usr/gen_init_cpio.c +81 -58
index edcdb8abfa31c..013f5668f33a8 100644
@@ -23,64 +23,71 @@
 #define xstr(s) #s
 #define str(s) xstr(s)
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#define CPIO_HDR_LEN 110
+#define padlen(_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align))
 
+static char padding[512];
 static unsigned int offset;
 static unsigned int ino = 721;
 static time_t default_mtime;
 static bool do_file_mtime;
 static bool do_csum = false;
+static int outfd = STDOUT_FILENO;
 
 struct file_handler {
 	const char *type;
 	int (*handler)(const char *line);
 };
 
-static void push_string(const char *name)
+static int push_string(const char *name)
 {
 	unsigned int name_len = strlen(name) + 1;
+	ssize_t l;
+
+	l = write(outfd, name, name_len);
+	if (l != name_len)
+		return -1;
 
-	fputs(name, stdout);
-	putchar(0);
 	offset += name_len;
+	return 0;
 }
 
-static void push_pad (void)
+static int push_pad(size_t padlen)
 {
-	while (offset & 3) {
-		putchar(0);
-		offset++;
-	}
+	ssize_t l = 0;
+
+	if (!padlen)
+		return 0;
+
+	if (padlen < sizeof(padding))
+		l = write(outfd, padding, padlen);
+	if (l != padlen)
+		return -1;
+
+	offset += padlen;
+	return 0;
 }
 
-static void push_rest(const char *name)
+static int push_rest(const char *name)
 {
 	unsigned int name_len = strlen(name) + 1;
-	unsigned int tmp_ofs;
+	ssize_t l;
 
-	fputs(name, stdout);
-	putchar(0);
-	offset += name_len;
+	l = write(outfd, name, name_len);
+	if (l != name_len)
+		return -1;
 
-	tmp_ofs = name_len + 110;
-	while (tmp_ofs & 3) {
-		putchar(0);
-		offset++;
-		tmp_ofs++;
-	}
-}
+	offset += name_len;
 
-static void push_hdr(const char *s)
-{
-	fputs(s, stdout);
-	offset += 110;
+	return push_pad(padlen(name_len + CPIO_HDR_LEN, 4));
 }
 
-static void cpio_trailer(void)
+static int cpio_trailer(void)
 {
-	char s[256];
 	const char name[] = "TRAILER!!!";
+	int l;
 
-	sprintf(s, "%s%08X%08X%08lX%08lX%08X%08lX"
+	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
 		0,			/* ino */
@@ -96,23 +103,24 @@ static void cpio_trailer(void)
 		0,			/* rminor */
 		(unsigned)strlen(name)+1, /* namesize */
 		0);			/* chksum */
-	push_hdr(s);
-	push_rest(name);
+	offset += l;
 
-	while (offset % 512) {
-		putchar(0);
-		offset++;
-	}
+	if (l != CPIO_HDR_LEN
+	 || push_rest(name) < 0
+	 || push_pad(padlen(offset, 512)) < 0)
+		return -1;
+
+	return 0;
 }
 
 static int cpio_mkslink(const char *name, const char *target,
 			 unsigned int mode, uid_t uid, gid_t gid)
 {
-	char s[256];
+	int l;
 
 	if (name[0] == '/')
 		name++;
-	sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX"
+	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
 		ino++,			/* ino */
@@ -128,12 +136,17 @@ static int cpio_mkslink(const char *name, const char *target,
 		0,			/* rminor */
 		(unsigned)strlen(name) + 1,/* namesize */
 		0);			/* chksum */
-	push_hdr(s);
-	push_string(name);
-	push_pad();
-	push_string(target);
-	push_pad();
+	offset += l;
+
+	if (l != CPIO_HDR_LEN
+	 || push_string(name) < 0
+	 || push_pad(padlen(offset, 4)) < 0
+	 || push_string(target) < 0
+	 || push_pad(padlen(offset, 4)) < 0)
+		return -1;
+
 	return 0;
+
 }
 
 static int cpio_mkslink_line(const char *line)
@@ -157,11 +170,11 @@ static int cpio_mkslink_line(const char *line)
 static int cpio_mkgeneric(const char *name, unsigned int mode,
 		       uid_t uid, gid_t gid)
 {
-	char s[256];
+	int l;
 
 	if (name[0] == '/')
 		name++;
-	sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX"
+	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
 		ino++,			/* ino */
@@ -177,8 +190,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,
 		0,			/* rminor */
 		(unsigned)strlen(name) + 1,/* namesize */
 		0);			/* chksum */
-	push_hdr(s);
-	push_rest(name);
+	offset += l;
+
+	if (l != CPIO_HDR_LEN
+	 || push_rest(name) < 0)
+		return -1;
+
 	return 0;
 }
 
@@ -246,7 +263,7 @@ static int cpio_mknod(const char *name, unsigned int mode,
 		       uid_t uid, gid_t gid, char dev_type,
 		       unsigned int maj, unsigned int min)
 {
-	char s[256];
+	int l;
 
 	if (dev_type == 'b')
 		mode |= S_IFBLK;
@@ -255,7 +272,7 @@ static int cpio_mknod(const char *name, unsigned int mode,
 
 	if (name[0] == '/')
 		name++;
-	sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX"
+	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
 		ino++,			/* ino */
@@ -271,8 +288,12 @@ static int cpio_mknod(const char *name, unsigned int mode,
 		min,			/* rminor */
 		(unsigned)strlen(name) + 1,/* namesize */
 		0);			/* chksum */
-	push_hdr(s);
-	push_rest(name);
+	offset += l;
+
+	if (l != CPIO_HDR_LEN
+	 || push_rest(name) < 0)
+		return -1;
+
 	return 0;
 }
 
@@ -324,11 +345,9 @@ static int cpio_mkfile(const char *name, const char *location,
 			unsigned int mode, uid_t uid, gid_t gid,
 			unsigned int nlinks)
 {
-	char s[256];
 	struct stat buf;
 	unsigned long size;
-	int file;
-	int retval;
+	int file, retval, l;
 	int rc = -1;
 	time_t mtime;
 	int namesize;
@@ -386,7 +405,7 @@ static int cpio_mkfile(const char *name, const char *location,
 		if (name[0] == '/')
 			name++;
 		namesize = strlen(name) + 1;
-		sprintf(s,"%s%08X%08X%08lX%08lX%08X%08lX"
+		l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 		       "%08lX%08X%08X%08X%08X%08X%08X",
 			do_csum ? "070702" : "070701", /* magic */
 			ino,			/* ino */
@@ -402,9 +421,12 @@ static int cpio_mkfile(const char *name, const char *location,
 			0,			/* rminor */
 			namesize,		/* namesize */
 			size ? csum : 0);	/* chksum */
-		push_hdr(s);
-		push_string(name);
-		push_pad();
+		offset += l;
+
+		if (l != CPIO_HDR_LEN
+		 || push_string(name) < 0
+		 || push_pad(padlen(offset, 4)) < 0)
+			goto error;
 
 		while (size) {
 			unsigned char filebuf[65536];
@@ -417,14 +439,15 @@ static int cpio_mkfile(const char *name, const char *location,
 				goto error;
 			}
 
-			if (fwrite(filebuf, this_read, 1, stdout) != 1) {
+			if (write(outfd, filebuf, this_read) != this_read) {
 				fprintf(stderr, "writing filebuf failed\n");
 				goto error;
 			}
 			offset += this_read;
 			size -= this_read;
 		}
-		push_pad();
+		if (push_pad(padlen(offset, 4)) < 0)
+			goto error;
 
 		name += namesize;
 	}
@@ -691,7 +714,7 @@ int main (int argc, char *argv[])
 		}
 	}
 	if (ec == 0)
-		cpio_trailer();
+		ec = cpio_trailer();
 
 	exit(ec);
 }
Signature · 1 line
2.43.0
#2 lore raw

RFC PATCH 2/5 gen_init_cpio: support -o <output_path> parameter

In reply to David Disseldorp

This is another preparatory change to allow for reflink-optimized cpio archives with file data written / cloned via copy_file_range(). The output file is truncated prior to write, so that it maps to usr/gen_initramfs.sh usage. It may make sense to offer an append option in future, for easier archive concatenation.

---
 usr/gen_init_cpio.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)
usr/gen_init_cpio.c +15 -4
index 013f5668f33a8..539b124d75416 100644
@@ -110,7 +110,7 @@ static int cpio_trailer(void)
 	 || push_pad(padlen(offset, 512)) < 0)
 		return -1;
 
-	return 0;
+	return fsync(outfd);
 }
 
 static int cpio_mkslink(const char *name, const char *target,
@@ -532,7 +532,7 @@ static int cpio_mkfile_line(const char *line)
 static void usage(const char *prog)
 {
 	fprintf(stderr, "Usage:\n"
-		"\t%s [-t <timestamp>] [-c] <cpio_list>\n"
+		"\t%s [-t <timestamp>] [-c] [-o <output_path>] <cpio_list>\n"
 		"\n"
 		"<cpio_list> is a file containing newline separated entries that\n"
 		"describe the files to be included in the initramfs archive:\n"
@@ -569,7 +569,8 @@ static void usage(const char *prog)
 		"as mtime for symlinks, directories, regular and special files.\n"
 		"The default is to use the current time for all files, but\n"
 		"preserve modification time for regular files.\n"
-		"-c: calculate and store 32-bit checksums for file data.\n",
+		"-c: calculate and store 32-bit checksums for file data.\n"
+		"<output_path>: write cpio to this file instead of stdout\n",
 		prog);
 }
 
@@ -611,7 +612,7 @@ int main (int argc, char *argv[])
 
 	default_mtime = time(NULL);
 	while (1) {
-		int opt = getopt(argc, argv, "t:ch");
+		int opt = getopt(argc, argv, "t:cho:");
 		char *invalid;
 
 		if (opt == -1)
@@ -630,6 +631,16 @@ int main (int argc, char *argv[])
 		case 'c':
 			do_csum = true;
 			break;
+		case 'o':
+			outfd = open(optarg,
+				     O_WRONLY | O_CREAT | O_LARGEFILE | O_TRUNC,
+				     0600);
+			if (outfd < 0) {
+				fprintf(stderr, "failed to open %s\n", optarg);
+				usage(argv[0]);
+				exit(1);
+			}
+			break;
 		case 'h':
 		case '?':
 			usage(argv[0]);
Signature · 1 line
2.43.0
#3 lore raw

RFC PATCH 3/5 gen_init_cpio: attempt copy_file_range for file data

In reply to David Disseldorp

The copy_file_range() syscall can improve copy performance by cloning extents between cpio archive source and destination files. Existing read / write based copy logic is retained for fallback in case the copy_file_range() syscall is unsupported or unavailable due to cross-filesystem EXDEV, etc.

Clone, as opposed to copy, of source file extents into the output cpio archive may (e.g. on Btrfs) require alignment of the output to the filesystem block size. This could be achieved by inserting padding entries into the cpio archive manifest. In future it may be worth zero-padding the filename field similar to dracut-cpio[1].

---
 usr/gen_init_cpio.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)
usr/gen_init_cpio.c +11 -1
index 539b124d75416..c8edb512af215 100644
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -353,6 +354,7 @@ static int cpio_mkfile(const char *name, const char *location,
 	int namesize;
 	unsigned int i;
 	uint32_t csum = 0;
+	ssize_t this_read;
 
 	mode |= S_IFREG;
 
@@ -428,9 +430,17 @@ static int cpio_mkfile(const char *name, const char *location,
 		 || push_pad(padlen(offset, 4)) < 0)
 			goto error;
 
+		if (size) {
+			this_read = copy_file_range(file, NULL, outfd, NULL, size, 0);
+			if (this_read > 0 && this_read <= size) {
+				offset += this_read;
+				size -= this_read;
+			}
+			/* short or failed copy falls back to read/write... */
+		}
+
 		while (size) {
 			unsigned char filebuf[65536];
-			ssize_t this_read;
 			size_t this_size = MIN(size, sizeof(filebuf));
 
 			this_read = read(file, filebuf, this_size);
Signature · 1 line
2.43.0
#4 lore raw

RFC PATCH 4/5 gen_init_cpio: avoid duplicate strlen calls

In reply to David Disseldorp

We determine the filename length for the cpio header, so shouldn't recalculate it when writing out the filename.

---
 usr/gen_init_cpio.c | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)
usr/gen_init_cpio.c +24 -16
index c8edb512af215..5f86550268719 100644
@@ -25,6 +25,7 @@
 #define str(s) xstr(s)
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define CPIO_HDR_LEN 110
+#define CPIO_TRAILER "TRAILER!!!"
 #define padlen(_off, _align) (((_align) - ((_off) & ((_align) - 1))) % (_align))
 
 static char padding[512];
@@ -40,9 +41,8 @@ struct file_handler {
 	int (*handler)(const char *line);
 };
 
-static int push_string(const char *name)
+static int push_buf(const char *name, size_t name_len)
 {
-	unsigned int name_len = strlen(name) + 1;
 	ssize_t l;
 
 	l = write(outfd, name, name_len);
@@ -69,9 +69,8 @@ static int push_pad(size_t padlen)
 	return 0;
 }
 
-static int push_rest(const char *name)
+static int push_rest(const char *name, size_t name_len)
 {
-	unsigned int name_len = strlen(name) + 1;
 	ssize_t l;
 
 	l = write(outfd, name, name_len);
@@ -85,8 +84,8 @@ static int push_rest(const char *name)
 
 static int cpio_trailer(void)
 {
-	const char name[] = "TRAILER!!!";
 	int l;
+	unsigned int namesize = sizeof(CPIO_TRAILER);
 
 	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
@@ -102,12 +101,12 @@ static int cpio_trailer(void)
 		0,			/* minor */
 		0,			/* rmajor */
 		0,			/* rminor */
-		(unsigned)strlen(name)+1, /* namesize */
+		namesize,		/* namesize */
 		0);			/* chksum */
 	offset += l;
 
 	if (l != CPIO_HDR_LEN
-	 || push_rest(name) < 0
+	 || push_rest(CPIO_TRAILER, namesize) < 0
 	 || push_pad(padlen(offset, 512)) < 0)
 		return -1;
 
@@ -118,9 +117,12 @@ static int cpio_mkslink(const char *name, const char *target,
 			 unsigned int mode, uid_t uid, gid_t gid)
 {
 	int l;
+	unsigned int namesize, targetsize = strlen(target) + 1;
 
 	if (name[0] == '/')
 		name++;
+	namesize = strlen(name) + 1;
+
 	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
@@ -130,19 +132,19 @@ static int cpio_mkslink(const char *name, const char *target,
 		(long) gid,		/* gid */
 		1,			/* nlink */
 		(long) default_mtime,	/* mtime */
-		(unsigned)strlen(target)+1, /* filesize */
+		targetsize,		/* filesize */
 		3,			/* major */
 		1,			/* minor */
 		0,			/* rmajor */
 		0,			/* rminor */
-		(unsigned)strlen(name) + 1,/* namesize */
+		namesize,		/* namesize */
 		0);			/* chksum */
 	offset += l;
 
 	if (l != CPIO_HDR_LEN
-	 || push_string(name) < 0
+	 || push_buf(name, namesize) < 0
 	 || push_pad(padlen(offset, 4)) < 0
-	 || push_string(target) < 0
+	 || push_buf(target, targetsize) < 0
 	 || push_pad(padlen(offset, 4)) < 0)
 		return -1;
 
@@ -172,9 +174,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,
 		       uid_t uid, gid_t gid)
 {
 	int l;
+	unsigned int namesize;
 
 	if (name[0] == '/')
 		name++;
+	namesize = strlen(name) + 1;
+
 	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
@@ -189,12 +194,12 @@ static int cpio_mkgeneric(const char *name, unsigned int mode,
 		1,			/* minor */
 		0,			/* rmajor */
 		0,			/* rminor */
-		(unsigned)strlen(name) + 1,/* namesize */
+		namesize,		/* namesize */
 		0);			/* chksum */
 	offset += l;
 
 	if (l != CPIO_HDR_LEN
-	 || push_rest(name) < 0)
+	 || push_rest(name, namesize) < 0)
 		return -1;
 
 	return 0;
@@ -265,6 +270,7 @@ static int cpio_mknod(const char *name, unsigned int mode,
 		       unsigned int maj, unsigned int min)
 {
 	int l;
+	unsigned int namesize;
 
 	if (dev_type == 'b')
 		mode |= S_IFBLK;
@@ -273,6 +279,8 @@ static int cpio_mknod(const char *name, unsigned int mode,
 
 	if (name[0] == '/')
 		name++;
+	namesize = strlen(name) + 1;
+
 	l = dprintf(outfd, "%s%08X%08X%08lX%08lX%08X%08lX"
 	       "%08X%08X%08X%08X%08X%08X%08X",
 		do_csum ? "070702" : "070701", /* magic */
@@ -287,12 +295,12 @@ static int cpio_mknod(const char *name, unsigned int mode,
 		1,			/* minor */
 		maj,			/* rmajor */
 		min,			/* rminor */
-		(unsigned)strlen(name) + 1,/* namesize */
+		namesize,		/* namesize */
 		0);			/* chksum */
 	offset += l;
 
 	if (l != CPIO_HDR_LEN
-	 || push_rest(name) < 0)
+	 || push_rest(name, namesize) < 0)
 		return -1;
 
 	return 0;
@@ -426,7 +434,7 @@ static int cpio_mkfile(const char *name, const char *location,
 		offset += l;
 
 		if (l != CPIO_HDR_LEN
-		 || push_string(name) < 0
+		 || push_buf(name, namesize) < 0
 		 || push_pad(padlen(offset, 4)) < 0)
 			goto error;
 
Signature · 1 line
2.43.0
#5 lore raw

RFC PATCH 5/5 gen_initramfs.sh: use gen_init_cpio -o parameter

In reply to David Disseldorp

gen_init_cpio can now write to a file directly, so use it when gen_initramfs.sh is called with -o (e.g. usr/Makefile invocation).

---
 usr/gen_initramfs.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
usr/gen_initramfs.sh +4 -3
index 14b5782f961a8..7eba2fddf0ef2 100755
@@ -193,7 +193,8 @@ root_gid=0
 dep_list=
 timestamp=
 cpio_list=$(mktemp ${TMPDIR:-/tmp}/cpiolist.XXXXXX)
-output="/dev/stdout"
+# gen_init_cpio writes to stdout by default
+output=""
 
 trap "rm -f $cpio_list" EXIT
 
@@ -207,7 +208,7 @@ while [ $# -gt 0 ]; do
 			shift
 			;;
 		"-o")	# generate cpio image named $1
-			output="$1"
+			output="-o $1"
 			shift
 			;;
 		"-u")	# map $1 to uid=0 (root)
@@ -246,4 +247,4 @@ done
 
 # If output_file is set we will generate cpio archive
 # we are careful to delete tmp files
-usr/gen_init_cpio $timestamp $cpio_list > $output
+usr/gen_init_cpio $output $timestamp $cpio_list
Signature · 1 line
2.43.0