Skip to contentSkip to conversation

mkfs: add ability to populate filesystem from directory

PATCH Superseded Luca Di Maio 2 participants 9 messages Apr 23, 2025

v4 of 12 Superseded Latest is v12 →

lore
Share & tools
lore
#0 lore raw

v4 cover mkfs: add ability to populate filesystem from directory

Currently the only way to pre populate an XFS partition is via the prototype file. While it works it has some limitations like:
- not allowed spaces in file names
- not preserving timestamps of original inodes

This series adds a new -P option to mkfs.xfs that allows users to populate a newly created filesystem directly from an existing directory. While similar to the prototype functionality, this doesn't require writing a prototype file.
The implementation preserves file and directory attributes (ownership, permissions, timestamps) from the source directory when copying content to the new filesystem.

[v1] -> [v2]
remove changes to protofile spec
ensure backward compatibility
[v2] -> [v3]
use inode_set_[acm]time() as suggested
avoid copying atime and ctime
they are often problematic for reproducibility, and mtime is the important information to preserve anyway
[v3] -> [v4]
rewrite functionality to populate directly from an input directory this is similar to mkfs.ext4 option.

Luca Di Maio (4):
proto: expose more functions from proto
mkfs: add -P flag to populate a filesystem from a directory
man: document -P flag to populate a filesystem from a directory
populate: add ability to populate a filesystem from a directory

 man/man8/mkfs.xfs.8.in |   7 +
 mkfs/Makefile          |   2 +-
 mkfs/populate.c        | 287 +++++++++++++++++++++++++++++++++++++++++
 mkfs/populate.h        |   4 +
 mkfs/proto.c           |  33 ++---
 mkfs/proto.h           |  22 ++++
 mkfs/xfs_mkfs.c        |  22 +++-
 7 files changed, 352 insertions(+), 25 deletions(-)
 create mode 100644 mkfs/populate.c
 create mode 100644 mkfs/populate.h

--
2.49.0

#1 lore raw

PATCH v4 1/4 proto: expose more functions from proto

In order to minimize code duplication, we want to expose more functions from proto. This will simplify alternative implementations of filesystem population.

---
 mkfs/proto.c | 33 ++++++++++++---------------------
 mkfs/proto.h | 22 ++++++++++++++++++++++
 2 files changed, 34 insertions(+), 21 deletions(-)
mkfs/proto.c +12 -21
index 6dd3a20..d5122e0 100644
@@ -16,11 +16,7 @@
  */
 static char *getstr(char **pp);
 static void fail(char *msg, int i);
-static struct xfs_trans * getres(struct xfs_mount *mp, uint blocks);
-static void rsvfile(xfs_mount_t *mp, xfs_inode_t *ip, long long len);
 static int newregfile(char **pp, char **fname);
-static void rtinit(xfs_mount_t *mp);
-static long filesize(int fd);
 static int slashes_are_spaces;
 
 /*
@@ -115,7 +111,7 @@ res_failed(
 	fail(_("cannot reserve space"), i);
 }
 
-static struct xfs_trans *
+struct xfs_trans *
 getres(
 	struct xfs_mount *mp,
 	uint		blocks)
@@ -196,7 +192,7 @@ getdirentname(
 	return p;
 }
 
-static void
+void
 rsvfile(
 	xfs_mount_t	*mp,
 	xfs_inode_t	*ip,
@@ -242,7 +238,7 @@ rsvfile(
 		fail(_("committing space for a file failed"), error);
 }
 
-static void
+void
 writesymlink(
 	struct xfs_trans	*tp,
 	struct xfs_inode	*ip,
@@ -303,7 +299,7 @@ writefile_range(
 	}
 }
 
-static void
+void
 writefile(
 	struct xfs_inode	*ip,
 	const char		*fname,
@@ -410,7 +406,7 @@ writeattr(
 		fail(_("setting xattr value"), error);
 }
 
-static void
+void
 writeattrs(
 	struct xfs_inode	*ip,
 	const char		*fname,
@@ -467,7 +463,7 @@ newregfile(
 	return fd;
 }
 
-static void
+void
 newdirent(
 	struct xfs_mount	*mp,
 	struct xfs_trans	*tp,
@@ -498,7 +494,7 @@ newdirent(
 	}
 }
 
-static void
+void
 newdirectory(
 	xfs_mount_t	*mp,
 	xfs_trans_t	*tp,
@@ -512,7 +508,7 @@ newdirectory(
 		fail(_("directory create error"), error);
 }
 
-static struct xfs_parent_args *
+struct xfs_parent_args *
 newpptr(
 	struct xfs_mount	*mp)
 {
@@ -526,12 +522,7 @@ newpptr(
 	return ret;
 }
 
-struct cred {
-	uid_t			cr_uid;
-	gid_t			cr_gid;
-};
-
-static int
+int
 creatproto(
 	struct xfs_trans	**tpp,
 	struct xfs_inode	*dp,
@@ -594,7 +585,7 @@ creatproto(
 }
 
 /* Create a new metadata root directory. */
-static int
+int
 create_metadir(
 	struct xfs_mount	*mp)
 {
@@ -1122,7 +1113,7 @@ rtinit_groups(
 /*
  * Allocate the realtime bitmap and summary inodes, and fill in data if any.
  */
-static void
+void
 rtinit(
 	struct xfs_mount	*mp)
 {
@@ -1132,7 +1123,7 @@ rtinit(
 		rtinit_nogroups(mp);
 }
 
-static off_t
+off_t
 filesize(
 	int		fd)
 {
mkfs/proto.h +22 -0
index be1ceb4..8cb6674 100644
@@ -5,10 +5,32 @@
  */
 #ifndef MKFS_PROTO_H_
 #define MKFS_PROTO_H_
+struct cred {
+    uid_t cr_uid;
+    gid_t cr_gid;
+};
 
 char *setup_proto(char *fname);
 void parse_proto(struct xfs_mount *mp, struct fsxattr *fsx, char **pp,
 		int proto_slashes_are_spaces);
 void res_failed(int err);
+struct xfs_trans *getres(struct xfs_mount *mp, uint blocks);
+struct xfs_parent_args *newpptr(struct xfs_mount *mp);
+void writesymlink(struct xfs_trans *tp, struct xfs_inode *ip,
+                 char *buf, int len);
+void writefile(struct xfs_inode *ip, const char *fname, int fd);
+void writeattrs(struct xfs_inode *ip, const char *fname, int fd);
+void rsvfile(xfs_mount_t *mp, xfs_inode_t *ip, long long len);
+void newdirent(struct xfs_mount *mp, struct xfs_trans *tp,
+              struct xfs_inode *pip, struct xfs_name *name,
+              struct xfs_inode *ip, struct xfs_parent_args *ppargs);
+void newdirectory(xfs_mount_t *mp, xfs_trans_t *tp,
+                 xfs_inode_t *dp, xfs_inode_t *pdp);
+int creatproto(struct xfs_trans **tpp, struct xfs_inode *dp,
+              mode_t mode, xfs_dev_t rdev, struct cred *cr,
+              struct fsxattr *fsx, struct xfs_inode **ipp);
+long filesize(int fd);
+void rtinit(struct xfs_mount *mp);
+int create_metadir(struct xfs_mount *mp);
 
 #endif /* MKFS_PROTO_H_ */

--
2.49.0

#2 lore raw

Re: [PATCH v4 1/4] proto: expose more functions from proto

Quoted 2 lines · Luca Di Maio

In order to minimize code duplication, we want to expose more functions from proto. This will simplify alternative implementations of filesystem population.

Nit: please cap lines in the commit message to 73 characters.

I wonder if these helpers should instead move to a separate new source file. But let's leave that lingering for now until we're fully reviewed the functional changes.

#3 lore raw

PATCH v4 2/4 mkfs: add -P flag to populate a filesystem from a directory

In reply to Luca Di Maio

Add a -P flag to populate a newly created filesystem from a directory. This flag will be mutually exclusive with the -p prototype flag.

---
 mkfs/xfs_mkfs.c | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)
mkfs/xfs_mkfs.c +19 -3
index f5556fc..06d1c12 100644
@@ -14,6 +14,7 @@
 #include "libfrog/dahashselftest.h"
 #include "libfrog/fsproperties.h"
 #include "proto.h"
+#include "populate.h"
 #include <ini.h>
 
 #define TERABYTES(count, blog)	((uint64_t)(count) << (40 - (blog)))
@@ -1022,6 +1023,7 @@ struct cli_params {
 
 	char	*cfgfile;
 	char	*protofile;
+	char	*directory;
 
 	enum fsprop_autofsck autofsck;
 
@@ -5110,7 +5112,7 @@ main(
 	memcpy(&cli.sb_feat, &dft.sb_feat, sizeof(cli.sb_feat));
 	memcpy(&cli.fsx, &dft.fsx, sizeof(cli.fsx));
 
-	while ((c = getopt_long(argc, argv, "b:c:d:i:l:L:m:n:KNp:qr:s:CfV",
+	while ((c = getopt_long(argc, argv, "b:c:d:i:l:L:P:m:n:KNp:qr:s:CfV",
 					long_options, &option_index)) != EOF) {
 		switch (c) {
 		case 0:
@@ -5136,6 +5138,11 @@ main(
 				illegal(optarg, "L");
 			cfg.label = optarg;
 			break;
+		case 'P':
+			if (strlen(optarg) > sizeof(sbp->sb_fname))
+				illegal(optarg, "P");
+			cli.directory = optarg;
+			break;
 		case 'N':
 			dry_run = 1;
 			break;
@@ -5334,9 +5341,18 @@ main(
 		initialise_ag_freespace(mp, agno, worst_freelist);
 
 	/*
-	 * Allocate the root inode and anything else in the proto file.
+	 * Allocate the root inode and anything else in the proto file or source
+	 * directory.
+	 * Both functions will allocate the root inode, so we use them mutually.
 	 */
-	parse_proto(mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces);
+	if (cli.protofile && cli.directory) {
+		fprintf(stderr, "%s: error: specifying both -P and -p is not supported.\n", progname);
+		exit(1);
+	}
+	if (!cli.directory)
+		parse_proto(mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces);
+	else
+		populate_from_dir(mp, NULL, &cli.fsx, cli.directory);
 
 	/*
 	 * Protect ourselves against possible stupidity

--
2.49.0

#4 lore raw

Re: [PATCH v4 2/4] mkfs: add -P flag to populate a filesystem from a directory

mkfs/xfs_mkfs.c main( · line 5142

Quoted 9 lines · Luca Di Maio
mkfs/xfs_mkfs.c +2 -0
 					long_options, &option_index)) != EOF) {
 		switch (c) {
 		case 0:
@@ -5136,6 +5138,11 @@ main(
 				illegal(optarg, "L");
 			cfg.label = optarg;
 			break;
+		case 'P':
+			if (strlen(optarg) > sizeof(sbp->sb_fname))

This check looks weird. sb_fname is a 12-byte legacy file system name in the superblock that is rarely used and looks unrelated to this new feature.

mkfs/xfs_mkfs.c main( · line 5350

Quoted 3 lines
mkfs/xfs_mkfs.c +3 -0
+	if (cli.protofile && cli.directory) {
+		fprintf(stderr, "%s: error: specifying both -P and -p is not supported.\n", progname);
+		exit(1);

The usual XFS style is to keep very long log message on a line od their own. Also this needs to use the _() macro so that the string is included in the localization database:

		fprintf(stderr,
	_("%s: error: specifying both -P and -p is not supported.\n"),
			progname);

mkfs/xfs_mkfs.c main( · line 5355

Quoted 4 lines
mkfs/xfs_mkfs.c +4 -0
+	if (!cli.directory)
+		parse_proto(mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces);
+	else
+		populate_from_dir(mp, NULL, &cli.fsx, cli.directory);

Please keep the line length to 80 characters in the source code.

populate_from_dir is not added in this patch, did you forget to add newly added source files to the git repository? If so they are also missing from the makefile.

#5 lore raw

PATCH v4 3/4 man: document -P flag to populate a filesystem from a directory

In reply to Luca Di Maio

Document the -P flag into man page.

---
 man/man8/mkfs.xfs.8.in | 7 +++++++
 1 file changed, 7 insertions(+)
man/man8/mkfs.xfs.8.in +7 -0
index 37e3a88..507a2f9 100644
@@ -1291,6 +1291,13 @@ creating the file system.
 .B \-K
 Do not attempt to discard blocks at mkfs time.
 .TP
+.BI \-P " directory"
+Populate the filesystem starting from a
+.IR directory .
+This will copy the contents of the given directory into the root directory of
+the filesystem. Ownership, Extended attributes, Permissions and timestamps will
+be preserved from the source.
+.TP
 .B \-V
 Prints the version number and exits.
 .SH Configuration File Format

--
2.49.0

#6 lore raw

PATCH v4 4/4 populate: add ability to populate a filesystem from a directory

In reply to Luca Di Maio

This patch implements the functionality to populate a newly created XFS filesystem directly from an existing directory structure.

The population process steps are as follows:
- create the root inode before populating content
- recursively process nested directories
- handle regular files, directories, symlinks, character devices, block devices, and fifos
- preserve file and directory attributes (ownership, permissions)
- preserve timestamps from source files to maintain file history
- preserve file extended attributes

This functionality makes it easier to create populated filesystems without having to write protofiles manually.
It's particularly useful for reproducible builds.

---
 mkfs/Makefile   |   2 +-
 mkfs/populate.c | 287 ++++++++++++++++++++++++++++++++++++++++++++++++
 mkfs/populate.h |   4 +
 3 files changed, 292 insertions(+), 1 deletion(-)
 create mode 100644 mkfs/populate.c
 create mode 100644 mkfs/populate.h
mkfs/Makefile +1 -1
index 3d3f08a..75f3dcd 100644
@@ -9,7 +9,7 @@ LTCOMMAND = mkfs.xfs
 XFS_PROTOFILE = xfs_protofile
 
 HFILES =
-CFILES = proto.c xfs_mkfs.c
+CFILES = populate.c proto.c xfs_mkfs.c
 CFGFILES = \
 	dax_x86_64.conf \
 	lts_4.19.conf \
mkfs/populate.c new file +287 -0

287 lines not on this page · raw message

mkfs/populate.h new file +4 -0
new file mode 100644
index 0000000..e1b8587
@@ -0,0 +1,4 @@
+#ifndef MKFS_POPULATE_H_
+#define MKFS_POPULATE_H_
+void populate_from_dir(xfs_mount_t *mp, xfs_inode_t *pip, struct fsxattr *fsxp, char *name);
+#endif /* MKFS_POPULATE_H_ */

--
2.49.0

#7 lore raw

Re: [PATCH v4 4/4] populate: add ability to populate a filesystem from a directory

Ah, this is the new code.

Please reorder it to before wiring it up in mkfs, or even better just merge the two patches as that makes reviewing easier.

Review

Quoted 3 lines

+++ b/mkfs/populate.c

(quoted diff) +0 -0
@@ -0,0 +1,287 @@

Please add the SPDX license identifier and your or your employers copyright (whoever has the right to it) here.

mkfs/populate.c line 13

Quoted 3 lines
mkfs/populate.c new file +3 -0
+static void fail(char *msg, int i)
+{
+	fprintf(stderr, "%s: %s [%d - %s]\n", progname, msg, i, strerror(i));

This needs the _() treatment for localization.

mkfs/populate.c line 36

Quoted 6 lines
mkfs/populate.c new file +6 -0
+static void writetimestamps(struct xfs_inode *ip, struct stat statbuf)
+{
+	struct timespec64 ts;
+
+	/* Copy timestamps from source file to destination inode.
+	*  In order to not be influenced by our own access timestamp,

The usual kernel and xfprogs style for multi-line comments is:

	/*
	 * Copy timestamps from source file to destination inode.
	 * ..
	 */`

mkfs/populate.c line 51

Quoted 1 line
mkfs/populate.c new file +1 -0
+static void create_file(xfs_mount_t *mp, struct xfs_inode *pip,

We're (way too slowly) phasing out the use of typedefs for structs, so the xfs_mount_t above should be struct xfs_mount.

mkfs/populate.c line 61

Quoted 4 lines
mkfs/populate.c new file +4 -0
+	xfs_inode_t *ip;
+	xfs_trans_t *tp;
+	tp = getres(mp, 0);
+	ppargs = newpptr(mp);

Similar for the xfs_inode_t and xfs_trans_t above and a few more later in the patch.

Also please keep an empty line between delarations and the actual code for clarity.

Review line 15

Quoted 1 line
(quoted diff) +1 -0
+	// copy over timestamps

Please use /* */ style comments.

mkfs/populate.c line 100

Quoted 3 lines
mkfs/populate.c new file +3 -0
+	if ((dir = opendir(cur_path)) == NULL) {
+		fail(_("cannot open input dir"), 1);
+	}

No need to use braces for single line statements.

mkfs/populate.c line 111

Quoted 10 lines
mkfs/populate.c new file +10 -0
+	while ((entry = readdir(dir)) != NULL) {
+		char link_target[PATH_MAX];
+		char path[PATH_MAX];
+		int error;
+		int fd = -1;
+		int flags;
+		int majdev;
+		int mindev;
+		int mode;
+		off_t len;

Can you factor this quite huge loop body into a helper function?

mkfs/populate.h line 2

Quoted 6 lines

index 0000000..e1b8587

mkfs/populate.h new file +2 -0
@@ -0,0 +1,4 @@
+#ifndef MKFS_POPULATE_H_
+#define MKFS_POPULATE_H_

This also needs SPDX tag.

As you might have noticed this is really just nitpicking. The actual logic looks good to me.

#8 lore raw

Re: [PATCH v4 4/4] populate: add ability to populate a filesystem from a directory

Thanks Christoph for the review
I've addressed your suggestions in V5 patch set

L.