Skip to contentSkip to conversation

mkfs: add ability to populate filesystem from directory

PATCH Superseded Luca Di Maio 1 participant 5 messages Apr 23, 2025

v5 of 12 Superseded Latest is v12 →

lore
Share & tools
lore
#0 lore raw

v5 cover mkfs: add ability to populate filesystem from directory

Currently the only way to pre populate an XFS partition is via the prototype file. While it works it has some limitations like:
- not allowed spaces in file names
- not preserving timestamps of original inodes

This series adds a new -P option to mkfs.xfs that allows users to populate a newly created filesystem directly from an existing directory. While similar to the prototype functionality, this doesn't require writing a prototype file.
The implementation preserves file and directory attributes (ownership, permissions, timestamps) from the source directory when copying content to the new filesystem.

[v1] -> [v2]
remove changes to protofile spec
ensure backward compatibility
[v2] -> [v3]
use inode_set_[acm]time() as suggested
avoid copying atime and ctime
they are often problematic for reproducibility, and mtime is the important information to preserve anyway
[v3] -> [v4]
rewrite functionality to populate directly from an input directory this is similar to mkfs.ext4 option.
[v4] -> [v5]
reorder patch to make it easier to review
reflow to keep code below 80 chars
use _() macro in prints
add SPDX headers to new files
fix comment styling
move from typedef to structs
move direntry handling to own function

Luca Di Maio (4):
proto: expose more functions from proto
populate: add ability to populate a filesystem from a directory
mkfs: add -P flag to populate a filesystem from a directory
man: document -P flag to populate a filesystem from a directory

 man/man8/mkfs.xfs.8.in |   7 +
 mkfs/Makefile          |   2 +-
 mkfs/populate.c        | 313 +++++++++++++++++++++++++++++++++++++++++
 mkfs/populate.h        |  10 ++
 mkfs/proto.c           |  33 ++---
 mkfs/proto.h           |  22 +++
 mkfs/xfs_mkfs.c        |  23 ++-
 7 files changed, 385 insertions(+), 25 deletions(-)
 create mode 100644 mkfs/populate.c
 create mode 100644 mkfs/populate.h

--
2.49.0

#1 lore raw

PATCH v5 1/4 proto: expose more functions from proto

In order to minimize code duplication, we want to expose more functions from proto. This will simplify alternative implementations of filesystem population.

---
 mkfs/proto.c | 33 ++++++++++++---------------------
 mkfs/proto.h | 22 ++++++++++++++++++++++
 2 files changed, 34 insertions(+), 21 deletions(-)
mkfs/proto.c +12 -21
index 6dd3a20..d5122e0 100644
@@ -16,11 +16,7 @@
  */
 static char *getstr(char **pp);
 static void fail(char *msg, int i);
-static struct xfs_trans * getres(struct xfs_mount *mp, uint blocks);
-static void rsvfile(xfs_mount_t *mp, xfs_inode_t *ip, long long len);
 static int newregfile(char **pp, char **fname);
-static void rtinit(xfs_mount_t *mp);
-static long filesize(int fd);
 static int slashes_are_spaces;
 
 /*
@@ -115,7 +111,7 @@ res_failed(
 	fail(_("cannot reserve space"), i);
 }
 
-static struct xfs_trans *
+struct xfs_trans *
 getres(
 	struct xfs_mount *mp,
 	uint		blocks)
@@ -196,7 +192,7 @@ getdirentname(
 	return p;
 }
 
-static void
+void
 rsvfile(
 	xfs_mount_t	*mp,
 	xfs_inode_t	*ip,
@@ -242,7 +238,7 @@ rsvfile(
 		fail(_("committing space for a file failed"), error);
 }
 
-static void
+void
 writesymlink(
 	struct xfs_trans	*tp,
 	struct xfs_inode	*ip,
@@ -303,7 +299,7 @@ writefile_range(
 	}
 }
 
-static void
+void
 writefile(
 	struct xfs_inode	*ip,
 	const char		*fname,
@@ -410,7 +406,7 @@ writeattr(
 		fail(_("setting xattr value"), error);
 }
 
-static void
+void
 writeattrs(
 	struct xfs_inode	*ip,
 	const char		*fname,
@@ -467,7 +463,7 @@ newregfile(
 	return fd;
 }
 
-static void
+void
 newdirent(
 	struct xfs_mount	*mp,
 	struct xfs_trans	*tp,
@@ -498,7 +494,7 @@ newdirent(
 	}
 }
 
-static void
+void
 newdirectory(
 	xfs_mount_t	*mp,
 	xfs_trans_t	*tp,
@@ -512,7 +508,7 @@ newdirectory(
 		fail(_("directory create error"), error);
 }
 
-static struct xfs_parent_args *
+struct xfs_parent_args *
 newpptr(
 	struct xfs_mount	*mp)
 {
@@ -526,12 +522,7 @@ newpptr(
 	return ret;
 }
 
-struct cred {
-	uid_t			cr_uid;
-	gid_t			cr_gid;
-};
-
-static int
+int
 creatproto(
 	struct xfs_trans	**tpp,
 	struct xfs_inode	*dp,
@@ -594,7 +585,7 @@ creatproto(
 }
 
 /* Create a new metadata root directory. */
-static int
+int
 create_metadir(
 	struct xfs_mount	*mp)
 {
@@ -1122,7 +1113,7 @@ rtinit_groups(
 /*
  * Allocate the realtime bitmap and summary inodes, and fill in data if any.
  */
-static void
+void
 rtinit(
 	struct xfs_mount	*mp)
 {
@@ -1132,7 +1123,7 @@ rtinit(
 		rtinit_nogroups(mp);
 }
 
-static off_t
+off_t
 filesize(
 	int		fd)
 {
mkfs/proto.h +22 -0
index be1ceb4..8cb6674 100644
@@ -5,10 +5,32 @@
  */
 #ifndef MKFS_PROTO_H_
 #define MKFS_PROTO_H_
+struct cred {
+    uid_t cr_uid;
+    gid_t cr_gid;
+};
 
 char *setup_proto(char *fname);
 void parse_proto(struct xfs_mount *mp, struct fsxattr *fsx, char **pp,
 		int proto_slashes_are_spaces);
 void res_failed(int err);
+struct xfs_trans *getres(struct xfs_mount *mp, uint blocks);
+struct xfs_parent_args *newpptr(struct xfs_mount *mp);
+void writesymlink(struct xfs_trans *tp, struct xfs_inode *ip,
+                 char *buf, int len);
+void writefile(struct xfs_inode *ip, const char *fname, int fd);
+void writeattrs(struct xfs_inode *ip, const char *fname, int fd);
+void rsvfile(xfs_mount_t *mp, xfs_inode_t *ip, long long len);
+void newdirent(struct xfs_mount *mp, struct xfs_trans *tp,
+              struct xfs_inode *pip, struct xfs_name *name,
+              struct xfs_inode *ip, struct xfs_parent_args *ppargs);
+void newdirectory(xfs_mount_t *mp, xfs_trans_t *tp,
+                 xfs_inode_t *dp, xfs_inode_t *pdp);
+int creatproto(struct xfs_trans **tpp, struct xfs_inode *dp,
+              mode_t mode, xfs_dev_t rdev, struct cred *cr,
+              struct fsxattr *fsx, struct xfs_inode **ipp);
+long filesize(int fd);
+void rtinit(struct xfs_mount *mp);
+int create_metadir(struct xfs_mount *mp);
 
 #endif /* MKFS_PROTO_H_ */

--
2.49.0

#2 lore raw

PATCH v5 2/4 populate: add ability to populate a filesystem from a directory

In reply to Luca Di Maio

This patch implements the functionality to populate a newly created XFS filesystem directly from an existing directory structure.

The population process steps are as follows:
- create the root inode before populating content
- recursively process nested directories
- handle regular files, directories, symlinks, char devices, block devices, and fifos
- preserve file and directory attributes (ownership, permissions)
- preserve timestamps from source files to maintain file history
- preserve file extended attributes

This functionality makes it easier to create populated filesystems without having to write protofiles manually.
It's particularly useful for reproducible builds.

---
 mkfs/Makefile   |   2 +-
 mkfs/populate.c | 313 ++++++++++++++++++++++++++++++++++++++++++++++++
 mkfs/populate.h |  10 ++
 3 files changed, 324 insertions(+), 1 deletion(-)
 create mode 100644 mkfs/populate.c
 create mode 100644 mkfs/populate.h
mkfs/Makefile +1 -1
index 3d3f08a..75f3dcd 100644
@@ -9,7 +9,7 @@ LTCOMMAND = mkfs.xfs
 XFS_PROTOFILE = xfs_protofile
 
 HFILES =
-CFILES = proto.c xfs_mkfs.c
+CFILES = populate.c proto.c xfs_mkfs.c
 CFGFILES = \
 	dax_x86_64.conf \
 	lts_4.19.conf \
mkfs/populate.c new file +313 -0

313 lines not on this page · raw message

mkfs/populate.h new file +10 -0
new file mode 100644
index 0000000..d65df57
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2025 Chainguard, Inc.
+ * All Rights Reserved.
+ * Author: Luca Di Maio <[email protected]>
+ */
+#ifndef MKFS_POPULATE_H_
+#define MKFS_POPULATE_H_
+void populate_from_dir(xfs_mount_t *mp, xfs_inode_t *pip, struct fsxattr *fsxp, char *name);
+#endif /* MKFS_POPULATE_H_ */

--
2.49.0

#3 lore raw

PATCH v5 3/4 mkfs: add -P flag to populate a filesystem from a directory

In reply to Luca Di Maio

Add a -P flag to populate a newly created filesystem from a directory. This flag will be mutually exclusive with the -p prototype flag.

---
 mkfs/xfs_mkfs.c | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)
mkfs/xfs_mkfs.c +20 -3
index f5556fc..44a8d73 100644
@@ -14,6 +14,7 @@
 #include "libfrog/dahashselftest.h"
 #include "libfrog/fsproperties.h"
 #include "proto.h"
+#include "populate.h"
 #include <ini.h>
 
 #define TERABYTES(count, blog)	((uint64_t)(count) << (40 - (blog)))
@@ -1022,6 +1023,7 @@ struct cli_params {
 
 	char	*cfgfile;
 	char	*protofile;
+	char	*directory;
 
 	enum fsprop_autofsck autofsck;
 
@@ -1170,6 +1172,7 @@ usage( void )
 /* naming */		[-n size=num,version=2|ci,ftype=0|1,parent=0|1]]\n\
 /* no-op info only */	[-N]\n\
 /* prototype file */	[-p fname]\n\
+/* populate */		[-P directory]\n\
 /* quiet */		[-q]\n\
 /* realtime subvol */	[-r extsize=num,size=num,rtdev=xxx,rgcount=n,rgsize=n,\n\
 			    concurrency=num]\n\
@@ -5110,7 +5113,7 @@ main(
 	memcpy(&cli.sb_feat, &dft.sb_feat, sizeof(cli.sb_feat));
 	memcpy(&cli.fsx, &dft.fsx, sizeof(cli.fsx));
 
-	while ((c = getopt_long(argc, argv, "b:c:d:i:l:L:m:n:KNp:qr:s:CfV",
+	while ((c = getopt_long(argc, argv, "b:c:d:i:l:L:P:m:n:KNp:qr:s:CfV",
 					long_options, &option_index)) != EOF) {
 		switch (c) {
 		case 0:
@@ -5136,6 +5139,9 @@ main(
 				illegal(optarg, "L");
 			cfg.label = optarg;
 			break;
+		case 'P':
+			cli.directory = optarg;
+			break;
 		case 'N':
 			dry_run = 1;
 			break;
@@ -5334,9 +5340,20 @@ main(
 		initialise_ag_freespace(mp, agno, worst_freelist);
 
 	/*
-	 * Allocate the root inode and anything else in the proto file.
+	 * Allocate the root inode and anything else in the proto file or source
+	 * directory.
+	 * Both functions will allocate the root inode, so we use them mutually.
 	 */
-	parse_proto(mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces);
+	if (cli.protofile && cli.directory) {
+		fprintf(stderr,
+			_("%s: error: specifying both -P and -p is not supported\n"),
+			progname);
+		exit(1);
+	}
+	if (!cli.directory)
+		parse_proto(mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces);
+	else
+		populate_from_dir(mp, NULL, &cli.fsx, cli.directory);
 
 	/*
 	 * Protect ourselves against possible stupidity

--
2.49.0

#4 lore raw

PATCH v5 4/4 man: document -P flag to populate a filesystem from a directory

In reply to Luca Di Maio

Document the -P flag into man page.

---
 man/man8/mkfs.xfs.8.in | 7 +++++++
 1 file changed, 7 insertions(+)
man/man8/mkfs.xfs.8.in +7 -0
index 37e3a88..507a2f9 100644
@@ -1291,6 +1291,13 @@ creating the file system.
 .B \-K
 Do not attempt to discard blocks at mkfs time.
 .TP
+.BI \-P " directory"
+Populate the filesystem starting from a
+.IR directory .
+This will copy the contents of the given directory into the root directory of
+the filesystem. Ownership, Extended attributes, Permissions and timestamps will
+be preserved from the source.
+.TP
 .B \-V
 Prints the version number and exits.
 .SH Configuration File Format

--
2.49.0