mkfs: add ability to populate filesystem from directory
v4 cover mkfs: add ability to populate filesystem from directory
Currently the only way to pre populate an XFS partition is via the prototype file. While it works it has some limitations like:
- not allowed spaces in file names
- not preserving timestamps of original inodes
This series adds a new -P option to mkfs.xfs that allows users to populate a newly created filesystem directly from an existing directory. While similar to the prototype functionality, this doesn't require writing a prototype file.
The implementation preserves file and directory attributes (ownership, permissions, timestamps) from the source directory when copying content to the new filesystem.
[v1] -> [v2]
remove changes to protofile spec
ensure backward compatibility
[v2] -> [v3]
use inode_set_[acm]time() as suggested
avoid copying atime and ctime
they are often problematic for reproducibility, and mtime is the important information to preserve anyway
[v3] -> [v4]
rewrite functionality to populate directly from an input directory this is similar to mkfs.ext4 option.
Luca Di Maio (4):
proto: expose more functions from proto
mkfs: add -P flag to populate a filesystem from a directory
man: document -P flag to populate a filesystem from a directory
populate: add ability to populate a filesystem from a directory
man/man8/mkfs.xfs.8.in | 7 + mkfs/Makefile | 2 +- mkfs/populate.c | 287 +++++++++++++++++++++++++++++++++++++++++ mkfs/populate.h | 4 + mkfs/proto.c | 33 ++--- mkfs/proto.h | 22 ++++ mkfs/xfs_mkfs.c | 22 +++- 7 files changed, 352 insertions(+), 25 deletions(-) create mode 100644 mkfs/populate.c create mode 100644 mkfs/populate.h
--
2.49.0
PATCH v4 1/4 proto: expose more functions from proto
In order to minimize code duplication, we want to expose more functions from proto. This will simplify alternative implementations of filesystem population.
--- mkfs/proto.c | 33 ++++++++++++--------------------- mkfs/proto.h | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 21 deletions(-)
mkfs/proto.c +12 -21
@@ -16,11 +16,7 @@*/ static char *getstr (char **pp);static void fail (char *msg,int i); -static struct xfs_trans *getres (struct xfs_mount *mp,uint blocks); -static void rsvfile (xfs_mount_t *mp,xfs_inode_t *ip,long long len);static int newregfile (char **pp,char **fname); -static void rtinit (xfs_mount_t *mp); -static long filesize (int fd);static int slashes_are_spaces;/*
@@ -115,7 +111,7 @@ res_failed(fail (_ ("cannot reserve space" ), i); } -static struct xfs_trans * +struct xfs_trans *getres (struct xfs_mount *mp,uint blocks)
@@ -196,7 +192,7 @@ getdirentname(return p; } -static void +void rsvfile( xfs_mount_t *mp, xfs_inode_t *ip,
@@ -242,7 +238,7 @@ rsvfile(fail (_ ("committing space for a file failed" ), error); } -static void +void writesymlink( struct xfs_trans *tp, struct xfs_inode *ip,
@@ -303,7 +299,7 @@ writefile_range( } } -static void +void writefile( struct xfs_inode *ip, const char *fname,
@@ -410,7 +406,7 @@ writeattr(fail (_ ("setting xattr value" ), error); } -static void +void writeattrs( struct xfs_inode *ip, const char *fname,
@@ -467,7 +463,7 @@ newregfile(return fd; } -static void +void newdirent( struct xfs_mount *mp, struct xfs_trans *tp,
@@ -498,7 +494,7 @@ newdirent( } } -static void +void newdirectory( xfs_mount_t *mp, xfs_trans_t *tp,
@@ -512,7 +508,7 @@ newdirectory(fail (_ ("directory create error" ), error); } -static struct xfs_parent_args * +struct xfs_parent_args *newpptr (struct xfs_mount *mp) {
@@ -526,12 +522,7 @@ newpptr(return ret; } -struct cred { -uid_t cr_uid; -gid_t cr_gid; -}; - -static int +int creatproto( struct xfs_trans **tpp, struct xfs_inode *dp,
@@ -594,7 +585,7 @@ creatproto( }/* Create a new metadata root directory. */ -static int +int create_metadir (struct xfs_mount *mp) {
@@ -1122,7 +1113,7 @@ rtinit_groups(/* * Allocate the realtime bitmap and summary inodes, and fill in data if any. */ -static void +void rtinit (struct xfs_mount *mp) {
@@ -1132,7 +1123,7 @@ rtinit(rtinit_nogroups (mp); } -static off_t +off_t filesize (int fd) {
mkfs/proto.h +22 -0
@@ -5,10 +5,32 @@*/ #ifndef MKFS_PROTO_H_ #define MKFS_PROTO_H_ +struct cred { +uid_t cr_uid; +gid_t cr_gid; +};char *setup_proto (char *fname);void parse_proto (struct xfs_mount *mp,struct fsxattr *fsx,char **pp,int proto_slashes_are_spaces);void res_failed (int err); +struct xfs_trans *getres (struct xfs_mount *mp,uint blocks); +struct xfs_parent_args *newpptr (struct xfs_mount *mp); +void writesymlink (struct xfs_trans *tp,struct xfs_inode *ip, +char *buf,int len); +void writefile (struct xfs_inode *ip,const char *fname,int fd); +void writeattrs (struct xfs_inode *ip,const char *fname,int fd); +void rsvfile (xfs_mount_t *mp,xfs_inode_t *ip,long long len); +void newdirent (struct xfs_mount *mp,struct xfs_trans *tp, +struct xfs_inode *pip,struct xfs_name *name, +struct xfs_inode *ip,struct xfs_parent_args *ppargs); +void newdirectory (xfs_mount_t *mp,xfs_trans_t *tp, +xfs_inode_t *dp,xfs_inode_t *pdp); +int creatproto (struct xfs_trans **tpp,struct xfs_inode *dp, +mode_t mode,xfs_dev_t rdev,struct cred *cr, +struct fsxattr *fsx,struct xfs_inode **ipp); +long filesize (int fd); +void rtinit (struct xfs_mount *mp); +int create_metadir (struct xfs_mount *mp);#endif /* MKFS_PROTO_H_ */
--
2.49.0
Re: [PATCH v4 1/4] proto: expose more functions from proto
Quoted 2 lines · Luca Di Maio
In order to minimize code duplication, we want to expose more functions from proto. This will simplify alternative implementations of filesystem population.
Nit: please cap lines in the commit message to 73 characters.
I wonder if these helpers should instead move to a separate new source file. But let's leave that lingering for now until we're fully reviewed the functional changes.
PATCH v4 2/4 mkfs: add -P flag to populate a filesystem from a directory
In reply to Luca Di MaioAdd a -P flag to populate a newly created filesystem from a directory. This flag will be mutually exclusive with the -p prototype flag.
--- mkfs/xfs_mkfs.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-)
mkfs/xfs_mkfs.c +19 -3
@@ -14,6 +14,7 @@#include "libfrog/dahashselftest.h" #include "libfrog/fsproperties.h" #include "proto.h" +#include "populate.h" #include <ini.h> #define TERABYTES (count, blog) ((uint64_t)(count) << (40 - (blog)))
@@ -1022,6 +1023,7 @@ struct cli_params {char *cfgfile;char *protofile; +char *directory;enum fsprop_autofsck autofsck;
@@ -5110,7 +5112,7 @@ main(memcpy (&cli.sb_feat, &dft.sb_feat,sizeof (cli.sb_feat));memcpy (&cli.fsx, &dft.fsx,sizeof (cli.fsx)); -while ((c =getopt_long (argc, argv,"b:c:d:i:l:L:m:n:KNp:qr:s:CfV" , +while ((c =getopt_long (argc, argv,"b:c:d:i:l:L:P:m:n:KNp:qr:s:CfV" , long_options, &option_index)) !=EOF ) {switch (c) {case 0 :
@@ -5136,6 +5138,11 @@ main(illegal (optarg,"L" ); cfg.label = optarg;break ; +case 'P' : +if (strlen (optarg) >sizeof (sbp->sb_fname)) +illegal (optarg,"P" ); + cli.directory = optarg; +break ;case 'N' : dry_run =1 ;break ;
@@ -5334,9 +5341,18 @@ main(initialise_ag_freespace (mp, agno, worst_freelist);/* -* Allocate the root inode and anything else in the proto file. +* Allocate the root inode and anything else in the proto file or source +* directory. +* Both functions will allocate the root inode, so we use them mutually. */ -parse_proto (mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces); +if (cli.protofile && cli.directory) { +fprintf (stderr,"%s: error: specifying both -P and -p is not supported.\n" , progname); +exit (1 ); + } +if (!cli.directory) +parse_proto (mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces); +else +populate_from_dir (mp,NULL , &cli.fsx, cli.directory);/* * Protect ourselves against possible stupidity
--
2.49.0
Re: [PATCH v4 2/4] mkfs: add -P flag to populate a filesystem from a directory
mkfs/xfs_mkfs.c main( · line 5142
Quoted 9 lines · Luca Di Maio
mkfs/xfs_mkfs.c +2
-0long_options, &option_index)) != EOF) {switch (c) {case 0 :@@ -5136,6 +5138,11 @@ main(illegal (optarg,"L" ); cfg.label = optarg;break ; +case 'P' : +if (strlen (optarg) >sizeof (sbp->sb_fname))
This check looks weird. sb_fname is a 12-byte legacy file system name in the superblock that is rarely used and looks unrelated to this new feature.
mkfs/xfs_mkfs.c main( · line 5350
Quoted 3 lines
mkfs/xfs_mkfs.c +3
-0+if (cli.protofile && cli.directory) { +fprintf (stderr,"%s: error: specifying both -P and -p is not supported.\n" , progname); +exit (1 );
The usual XFS style is to keep very long log message on a line od their own. Also this needs to use the _() macro so that the string is included in the localization database:
fprintf(stderr,
_("%s: error: specifying both -P and -p is not supported.\n"),
progname);mkfs/xfs_mkfs.c main( · line 5355
Quoted 4 lines
mkfs/xfs_mkfs.c +4
-0+if (!cli.directory) +parse_proto (mp, &cli.fsx, &protostring, cli.proto_slashes_are_spaces); +else +populate_from_dir (mp,NULL , &cli.fsx, cli.directory);
Please keep the line length to 80 characters in the source code.
populate_from_dir is not added in this patch, did you forget to add newly added source files to the git repository? If so they are also missing from the makefile.
PATCH v4 3/4 man: document -P flag to populate a filesystem from a directory
In reply to Luca Di MaioDocument the -P flag into man page.
--- man/man8/mkfs.xfs.8.in | 7 +++++++ 1 file changed, 7 insertions(+)
man/man8/mkfs.xfs.8.in +7 -0
@@ -1291,6 +1291,13 @@ creating the file system. .B \-K Do not attempt to discard blocks at mkfs time. .TP +.BI \-P " directory" +Populate the filesystem starting from a +.IR directory . +This will copy the contents of the given directory into the root directory of +the filesystem. Ownership, Extended attributes, Permissions and timestamps will +be preserved from the source. +.TP .B \-V Prints the version number and exits. .SH Configuration File Format
--
2.49.0
PATCH v4 4/4 populate: add ability to populate a filesystem from a directory
In reply to Luca Di MaioThis patch implements the functionality to populate a newly created XFS filesystem directly from an existing directory structure.
The population process steps are as follows:
- create the root inode before populating content
- recursively process nested directories
- handle regular files, directories, symlinks, character devices, block devices, and fifos
- preserve file and directory attributes (ownership, permissions)
- preserve timestamps from source files to maintain file history
- preserve file extended attributes
This functionality makes it easier to create populated filesystems without having to write protofiles manually.
It's particularly useful for reproducible builds.
--- mkfs/Makefile | 2 +- mkfs/populate.c | 287 ++++++++++++++++++++++++++++++++++++++++++++++++ mkfs/populate.h | 4 + 3 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 mkfs/populate.c create mode 100644 mkfs/populate.h
mkfs/Makefile +1 -1
@@ -9,7 +9,7 @@ LTCOMMAND = mkfs.xfs XFS_PROTOFILE = xfs_protofile HFILES = -CFILES = proto.c xfs_mkfs.c +CFILES = populate.c proto.c xfs_mkfs.c CFGFILES = \ dax_x86_64.conf \ lts_4.19.conf \
mkfs/populate.c new file +287 -0
287 lines not on this page · raw message
mkfs/populate.h new file +4 -0
@@ -0,0 +1,4 @@ +#ifndef MKFS_POPULATE_H_ +#define MKFS_POPULATE_H_ +void populate_from_dir (xfs_mount_t *mp,xfs_inode_t *pip,struct fsxattr *fsxp,char *name); +#endif /* MKFS_POPULATE_H_ */
--
2.49.0
Re: [PATCH v4 4/4] populate: add ability to populate a filesystem from a directory
Ah, this is the new code.
Please reorder it to before wiring it up in mkfs, or even better just merge the two patches as that makes reviewing easier.
Review
Quoted 3 lines
+++
b/mkfs/populate.c(quoted diff) +0
-0@@ -0,0 +1,287 @@
Please add the SPDX license identifier and your or your employers copyright (whoever has the right to it) here.
mkfs/populate.c line 13
Quoted 3 lines
mkfs/populate.c new file +3
-0+static void fail (char *msg,int i) +{ +fprintf (stderr,"%s: %s [%d - %s]\n" , progname, msg, i,strerror (i));
This needs the _() treatment for localization.
mkfs/populate.c line 36
Quoted 6 lines
mkfs/populate.c new file +6
-0+static void writetimestamps (struct xfs_inode *ip,struct stat statbuf) +{ +struct timespec64 ts; + +/* Copy timestamps from source file to destination inode. +* In order to not be influenced by our own access timestamp,
The usual kernel and xfprogs style for multi-line comments is:
/* * Copy timestamps from source file to destination inode. * .. */`
mkfs/populate.c line 51
Quoted 1 line
mkfs/populate.c new file +1
-0+static void create_file(xfs_mount_t *mp, struct xfs_inode *pip,
We're (way too slowly) phasing out the use of typedefs for structs, so the xfs_mount_t above should be struct xfs_mount.
mkfs/populate.c line 61
Quoted 4 lines
mkfs/populate.c new file +4
-0+xfs_inode_t *ip; +xfs_trans_t *tp; + tp =getres (mp,0 ); + ppargs =newpptr (mp);
Similar for the xfs_inode_t and xfs_trans_t above and a few more later in the patch.
Also please keep an empty line between delarations and the actual code for clarity.
Review line 15
Quoted 1 line
(quoted diff) +1
-0+ // copy over timestamps
Please use /* */ style comments.
mkfs/populate.c line 100
Quoted 3 lines
mkfs/populate.c new file +3
-0+if ((dir =opendir (cur_path)) ==NULL ) { +fail (_ ("cannot open input dir" ),1 ); + }
No need to use braces for single line statements.
mkfs/populate.c line 111
Quoted 10 lines
mkfs/populate.c new file +10
-0+while ((entry =readdir (dir)) !=NULL ) { +char link_target[PATH_MAX ]; +char path[PATH_MAX ]; +int error; +int fd =-1 ; +int flags; +int majdev; +int mindev; +int mode; +off_t len;
Can you factor this quite huge loop body into a helper function?
mkfs/populate.h line 2
Quoted 6 lines
index 0000000..e1b8587
mkfs/populate.h new file +2
-0@@ -0,0 +1,4 @@ +#ifndef MKFS_POPULATE_H_ +#define MKFS_POPULATE_H_
This also needs SPDX tag.
As you might have noticed this is really just nitpicking. The actual logic looks good to me.
Re: [PATCH v4 4/4] populate: add ability to populate a filesystem from a directory
Thanks Christoph for the review
I've addressed your suggestions in V5 patch set
L.