Skip to contentSkip to conversation

erofs-utils: lib: split out several helpers in inode.c

PATCH New Gao Xiang 1 participant 2 messages Apr 6, 2024 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH 1/2 erofs-utils: lib: split out several helpers in inode.c

From: Yifan Zhao <[email protected]>

The following new helpers are added to prepare for the upcoming multi-threaded inter-file compression:
- erofs_mkfs_handle_{non,}directory;
- erofs_write_unencoded_file.

---
 lib/inode.c | 106 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 60 insertions(+), 46 deletions(-)
lib/inode.c +60 -46
index ba0419f..f419f3c 100644
@@ -477,12 +477,8 @@ static int write_uncompressed_file_from_fd(struct erofs_inode *inode, int fd)
 	return 0;
 }
 
-int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
+int erofs_write_unencoded_file(struct erofs_inode *inode, int fd, u64 fpos)
 {
-	int ret;
-
-	DBG_BUGON(!inode->i_size);
-
 	if (cfg.c_chunkbits) {
 		inode->u.chunkbits = cfg.c_chunkbits;
 		/* chunk indexes when explicitly specified */
@@ -492,7 +488,17 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
 		return erofs_blob_write_chunked_file(inode, fd, fpos);
 	}
 
+	/* fallback to all data uncompressed */
+	return write_uncompressed_file_from_fd(inode, fd);
+}
+
+int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
+{
+	DBG_BUGON(!inode->i_size);
+
 	if (cfg.c_compr_opts[0].alg && erofs_file_is_compressible(inode)) {
+		int ret;
+
 		ret = erofs_write_compressed_file(inode, fd, fpos);
 		if (!ret || ret != -ENOSPC)
 			return ret;
@@ -500,9 +506,8 @@ int erofs_write_file(struct erofs_inode *inode, int fd, u64 fpos)
 		if (lseek(fd, fpos, SEEK_SET) < 0)
 			return -errno;
 	}
-
 	/* fallback to all data uncompressed */
-	return write_uncompressed_file_from_fd(inode, fd);
+	return erofs_write_unencoded_file(inode, fd, fpos);
 }
 
 static int erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
@@ -1095,52 +1100,44 @@ static void erofs_fixup_meta_blkaddr(struct erofs_inode *rootdir)
 	rootdir->nid = (off - meta_offset) >> EROFS_ISLOTBITS;
 }
 
-static int erofs_mkfs_build_tree(struct erofs_inode *dir, struct list_head *dirs)
+static int erofs_mkfs_handle_nondirectory(struct erofs_inode *inode)
 {
-	int ret;
-	DIR *_dir;
-	struct dirent *dp;
-	struct erofs_dentry *d;
-	unsigned int nr_subdirs, i_nlink;
+	int ret = 0;
 
-	ret = erofs_scan_file_xattrs(dir);
-	if (ret < 0)
-		return ret;
+	if (S_ISLNK(inode->i_mode)) {
+		char *const symlink = malloc(inode->i_size);
 
-	ret = erofs_prepare_xattr_ibody(dir);
-	if (ret < 0)
-		return ret;
-
-	if (!S_ISDIR(dir->i_mode)) {
-		if (S_ISLNK(dir->i_mode)) {
-			char *const symlink = malloc(dir->i_size);
-
-			if (!symlink)
-				return -ENOMEM;
-			ret = readlink(dir->i_srcpath, symlink, dir->i_size);
-			if (ret < 0) {
-				free(symlink);
-				return -errno;
-			}
-			ret = erofs_write_file_from_buffer(dir, symlink);
+		if (!symlink)
+			return -ENOMEM;
+		ret = readlink(inode->i_srcpath, symlink, inode->i_size);
+		if (ret < 0) {
 			free(symlink);
-		} else if (dir->i_size) {
-			int fd = open(dir->i_srcpath, O_RDONLY | O_BINARY);
-			if (fd < 0)
-				return -errno;
-
-			ret = erofs_write_file(dir, fd, 0);
-			close(fd);
-		} else {
-			ret = 0;
+			return -errno;
 		}
-		if (ret)
-			return ret;
+		ret = erofs_write_file_from_buffer(inode, symlink);
+		free(symlink);
+	} else if (inode->i_size) {
+		int fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
 
-		erofs_prepare_inode_buffer(dir);
-		erofs_write_tail_end(dir);
-		return 0;
+		if (fd < 0)
+			return -errno;
+		ret = erofs_write_file(inode, fd, 0);
+		close(fd);
 	}
+	if (ret)
+		return ret;
+	erofs_prepare_inode_buffer(inode);
+	erofs_write_tail_end(inode);
+	return 0;
+}
+
+static int erofs_mkfs_handle_directory(struct erofs_inode *dir, struct list_head *dirs)
+{
+	DIR *_dir;
+	struct dirent *dp;
+	struct erofs_dentry *d;
+	unsigned int nr_subdirs, i_nlink;
+	int ret;
 
 	_dir = opendir(dir->i_srcpath);
 	if (!_dir) {
@@ -1252,6 +1249,23 @@ err_closedir:
 	return ret;
 }
 
+static int erofs_mkfs_build_tree(struct erofs_inode *inode, struct list_head *dirs)
+{
+	int ret;
+
+	ret = erofs_scan_file_xattrs(inode);
+	if (ret < 0)
+		return ret;
+
+	ret = erofs_prepare_xattr_ibody(inode);
+	if (ret < 0)
+		return ret;
+
+	if (!S_ISDIR(inode->i_mode))
+		return erofs_mkfs_handle_nondirectory(inode);
+	return erofs_mkfs_handle_directory(inode, dirs);
+}
+
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path)
 {
 	LIST_HEAD(dirs);
Signature · 1 line
2.39.3
author
#1 lore raw

PATCH 2/2 erofs-utils: lib: refine on-disk meta arrangement again

Most workloads like ls -R and tar -c traverse in depth-first mode. However, it still arranges sub-directory inodes more closely so that it isn't a simple reversion compared to pre-BFS old versions.

Also the build performance out of linux-6.1.53 source code is greatly improved by 91.2% (33.040s->2.861s) as well as the new image size is decreased by 0.0094% (120 KiB), which is minor.

---
 lib/inode.c | 192 +++++++++++++++++++++++++---------------------------
 1 file changed, 91 insertions(+), 101 deletions(-)
lib/inode.c +91 -101
index f419f3c..79626ab 100644
@@ -162,7 +162,8 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
 
 	strncpy(d->name, name, EROFS_NAME_LEN - 1);
 	d->name[EROFS_NAME_LEN - 1] = '\0';
-
+	d->inode = NULL;
+	d->type = EROFS_FT_UNKNOWN;
 	list_add_tail(&d->d_child, &parent->i_subdirs);
 	return d;
 }
@@ -1131,7 +1132,7 @@ static int erofs_mkfs_handle_nondirectory(struct erofs_inode *inode)
 	return 0;
 }
 
-static int erofs_mkfs_handle_directory(struct erofs_inode *dir, struct list_head *dirs)
+static int erofs_mkfs_handle_directory(struct erofs_inode *dir)
 {
 	DIR *_dir;
 	struct dirent *dp;
@@ -1147,18 +1148,28 @@ static int erofs_mkfs_handle_directory(struct erofs_inode *dir, struct list_head
 	}
 
 	nr_subdirs = 0;
+	i_nlink = 0;
 	while (1) {
+		char buf[PATH_MAX];
+		struct erofs_inode *inode;
+
 		/*
 		 * set errno to 0 before calling readdir() in order to
 		 * distinguish end of stream and from an error.
 		 */
 		errno = 0;
 		dp = readdir(_dir);
-		if (!dp)
-			break;
+		if (!dp) {
+			if (!errno)
+				break;
+			ret = -errno;
+			goto err_closedir;
+		}
 
-		if (is_dot_dotdot(dp->d_name))
+		if (is_dot_dotdot(dp->d_name)) {
+			++i_nlink;
 			continue;
+		}
 
 		/* skip if it's a exclude file */
 		if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
@@ -1169,70 +1180,28 @@ static int erofs_mkfs_handle_directory(struct erofs_inode *dir, struct list_head
 			ret = PTR_ERR(d);
 			goto err_closedir;
 		}
-		nr_subdirs++;
-	}
-
-	if (errno) {
-		ret = -errno;
-		goto err_closedir;
-	}
-	closedir(_dir);
-
-	ret = erofs_prepare_dir_file(dir, nr_subdirs);
-	if (ret)
-		return ret;
-
-	ret = erofs_prepare_inode_buffer(dir);
-	if (ret)
-		return ret;
-	dir->bh->op = &erofs_skip_write_bhops;
-
-	if (IS_ROOT(dir))
-		erofs_fixup_meta_blkaddr(dir);
-
-	i_nlink = 0;
-	list_for_each_entry(d, &dir->i_subdirs, d_child) {
-		char buf[PATH_MAX];
-		unsigned char ftype;
-		struct erofs_inode *inode;
-
-		if (is_dot_dotdot(d->name)) {
-			++i_nlink;
-			continue;
-		}
 
-		ret = snprintf(buf, PATH_MAX, "%s/%s",
-			       dir->i_srcpath, d->name);
-		if (ret < 0 || ret >= PATH_MAX) {
-			/* ignore the too long path */
-			goto fail;
-		}
+		ret = snprintf(buf, PATH_MAX, "%s/%s", dir->i_srcpath, d->name);
+		if (ret < 0 || ret >= PATH_MAX)
+			goto err_closedir;
 
 		inode = erofs_iget_from_path(buf, true);
-
 		if (IS_ERR(inode)) {
 			ret = PTR_ERR(inode);
-fail:
-			d->inode = NULL;
-			d->type = EROFS_FT_UNKNOWN;
-			return ret;
-		}
-
-		/* a hardlink to the existed inode */
-		if (inode->i_parent) {
-			++inode->i_nlink;
-		} else {
-			inode->i_parent = dir;
-			erofs_igrab(inode);
-			list_add_tail(&inode->i_subdirs, dirs);
+			goto err_closedir;
 		}
-		ftype = erofs_mode_to_ftype(inode->i_mode);
-		i_nlink += (ftype == EROFS_FT_DIR);
 		d->inode = inode;
-		d->type = ftype;
-		erofs_info("file %s/%s dumped (type %u)",
-			   dir->i_srcpath, d->name, d->type);
+		d->type = erofs_mode_to_ftype(inode->i_mode);
+		i_nlink += S_ISDIR(inode->i_mode);
+		erofs_dbg("file %s added (type %u)", buf, d->type);
+		nr_subdirs++;
 	}
+	closedir(_dir);
+
+	ret = erofs_prepare_dir_file(dir, nr_subdirs);
+	if (ret)
+		return ret;
+
 	/*
 	 * if there're too many subdirs as compact form, set nlink=1
 	 * rather than upgrade to use extented form instead.
@@ -1242,6 +1211,11 @@ fail:
 		dir->i_nlink = 1;
 	else
 		dir->i_nlink = i_nlink;
+
+	ret = erofs_prepare_inode_buffer(dir);
+	if (ret)
+		return ret;
+	dir->bh->op = &erofs_skip_write_bhops;
 	return 0;
 
 err_closedir:
@@ -1249,10 +1223,16 @@ err_closedir:
 	return ret;
 }
 
-static int erofs_mkfs_build_tree(struct erofs_inode *inode, struct list_head *dirs)
+static int erofs_mkfs_build_tree(struct erofs_inode *inode)
 {
+	char *trimmed;
 	int ret;
 
+	trimmed = erofs_trim_for_progressinfo(erofs_fspath(inode->i_srcpath),
+					      sizeof("Processing  ...") - 1);
+	erofs_update_progressinfo("Processing %s ...", trimmed);
+	free(trimmed);
+
 	ret = erofs_scan_file_xattrs(inode);
 	if (ret < 0)
 		return ret;
@@ -1262,60 +1242,70 @@ static int erofs_mkfs_build_tree(struct erofs_inode *inode, struct list_head *di
 		return ret;
 
 	if (!S_ISDIR(inode->i_mode))
-		return erofs_mkfs_handle_nondirectory(inode);
-	return erofs_mkfs_handle_directory(inode, dirs);
+		ret = erofs_mkfs_handle_nondirectory(inode);
+	else
+		ret = erofs_mkfs_handle_directory(inode);
+	erofs_info("file %s dumped", erofs_fspath(inode->i_srcpath));
+	return ret;
 }
 
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path)
 {
-	LIST_HEAD(dirs);
-	struct erofs_inode *inode, *root, *dumpdir;
+	struct erofs_inode *root, *dumpdir;
+	int err;
 
 	root = erofs_iget_from_path(path, true);
 	if (IS_ERR(root))
 		return root;
 
-	(void)erofs_igrab(root);
 	root->i_parent = root;	/* rootdir mark */
-	list_add(&root->i_subdirs, &dirs);
+	root->next_dirwrite = NULL;
+	(void)erofs_igrab(root);
+	dumpdir = root;
+
+	err = erofs_mkfs_build_tree(root);
+	if (err)
+		return ERR_PTR(err);
+	erofs_fixup_meta_blkaddr(root);
 
-	dumpdir = NULL;
 	do {
 		int err;
-		char *trimmed;
-
-		inode = list_first_entry(&dirs, struct erofs_inode, i_subdirs);
-		list_del(&inode->i_subdirs);
-		init_list_head(&inode->i_subdirs);
-
-		trimmed = erofs_trim_for_progressinfo(
-				erofs_fspath(inode->i_srcpath),
-				sizeof("Processing  ...") - 1);
-		erofs_update_progressinfo("Processing %s ...", trimmed);
-		free(trimmed);
-
-		err = erofs_mkfs_build_tree(inode, &dirs);
-		if (err) {
-			root = ERR_PTR(err);
-			break;
+		struct erofs_inode *dir = dumpdir;
+		/* used for adding sub-directories in reverse order due to FIFO */
+		struct erofs_inode *head, **last = &head;
+		struct erofs_dentry *d;
+
+		dumpdir = dir->next_dirwrite;
+		list_for_each_entry(d, &dir->i_subdirs, d_child) {
+			struct erofs_inode *inode = d->inode;
+
+			if (is_dot_dotdot(d->name))
+				continue;
+			if (inode->i_parent) {
+				++inode->i_nlink;
+			} else {
+				inode->i_parent = dir;
+
+				err = erofs_mkfs_build_tree(inode);
+				if (err) {
+					root = ERR_PTR(err);
+					break;
+				}
+				if (S_ISDIR(inode->i_mode)) {
+					*last = inode;
+					last = &inode->next_dirwrite;
+					(void)erofs_igrab(inode);
+				}
+			}
 		}
+		*last = dumpdir;	/* fixup the last (or the only) one */
+		dumpdir = head;
+		erofs_write_dir_file(dir);
+		erofs_write_tail_end(dir);
+		dir->bh->op = &erofs_write_inode_bhops;
+		erofs_iput(dir);
+	} while (dumpdir);
 
-		if (S_ISDIR(inode->i_mode)) {
-			inode->next_dirwrite = dumpdir;
-			dumpdir = inode;
-		} else {
-			erofs_iput(inode);
-		}
-	} while (!list_empty(&dirs));
-
-	while (dumpdir) {
-		inode = dumpdir;
-		erofs_write_dir_file(inode);
-		erofs_write_tail_end(inode);
-		inode->bh->op = &erofs_write_inode_bhops;
-		dumpdir = inode->next_dirwrite;
-		erofs_iput(inode);
-	}
 	return root;
 }
 
Signature · 1 line
2.39.3