Skip to contentSkip to conversation

erofs-utils: add ERR_CAST macro

PATCH Superseded Jingbo Xu 2 participants 3 messages Jul 11, 2023 – Jul 13, 2023 Library code

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Jingbo Xu

author
#0 lore raw

PATCH 1/2 erofs-utils: add ERR_CAST macro

Add ERR_CAST macro to prepare for the upcoming tarerofs feature.

---
 include/erofs/err.h | 6 ++++++
 1 file changed, 6 insertions(+)
include/erofs/err.h +6 -0
index 08b0bdb..2ae9e21 100644
@@ -33,6 +33,12 @@ static inline long PTR_ERR(const void *ptr)
 	return (long) ptr;
 }
 
+static inline void * ERR_CAST(const void *ptr)
+{
+	/* cast away the const */
+	return (void *) ptr;
+}
+
 #ifdef __cplusplus
 }
 #endif
Signature · 1 line
2.19.1.6.gb485710b
author
#1 lore raw

PATCH 2/2 erofs-utils: introduce tarerofs

From: Gao Xiang <[email protected]>

Let's try to add a new mode "tarerofs" for mkfs.erofs.

It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full EROFS image [--tar=f];

- Generate an EROFS manifest image to reuse tar data [--tar=i], which also enables EROFS 512-byte blocks.

The second use case is mainly prepared for OCI direct mount without OCI blob unpacking. This also adds another --aufs option to transform aufs special files into overlayfs metadata.

[ Note that --tar=f generates lots of temporary files for now which
can impact performance since the original tar stream(s) may be non-seekable. ]

---
change log:
1. don't return prematurely for special device files (in erofs_mkfs_build_tree()). Otherwise erofs_mkfs_build_tree() may mistakenly return a positive integer number returned from erofs_prepare_xattr_ibody(), and mistakenly make it as the pointer to the root inode, which will cause mkfs.erofs segmentation fault.
2. automatically create parent directory if it's not included as an entry in the tarball stream. (refer to tarerofs_init_default_dir())
3. set device number for char and block device files (in tarerofs_parse_tar()).

---
 configure.ac              |   1 +
 include/erofs/blobchunk.h |   4 +-
 include/erofs/inode.h     |  12 +
 include/erofs/internal.h  |   7 +-
 include/erofs/tar.h       |  29 ++
 include/erofs/xattr.h     |   4 +
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |  47 ++-
 lib/inode.c               | 193 ++++++---
 lib/tar.c                 | 803 ++++++++++++++++++++++++++++++++++++++
 lib/xattr.c               |  46 ++-
 mkfs/main.c               | 134 +++++--
 12 files changed, 1177 insertions(+), 106 deletions(-)
 create mode 100644 include/erofs/tar.h
 create mode 100644 lib/tar.c
configure.ac +1 -0
index 2ade490..d23a68a 100644
@@ -126,6 +126,7 @@ AC_CHECK_HEADERS(m4_flatten([
 	fcntl.h
 	getopt.h
 	inttypes.h
+	linux/aufs_type.h
 	linux/falloc.h
 	linux/fs.h
 	linux/types.h
include/erofs/blobchunk.h +3 -1
index 49cb7bf..4269d82 100644
@@ -14,8 +14,10 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(erofs_off_t chunksize,
+			unsigned int device_id, erofs_blk_t blkaddr);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
-int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int erofs_blob_remap(void);
 void erofs_blob_exit(void);
 int erofs_blob_init(const char *blobfile_path);
include/erofs/inode.h +12 -0
index 058a235..e8a5670 100644
@@ -15,11 +15,23 @@ extern "C"
 
 #include "erofs/internal.h"
 
+static inline struct erofs_inode *erofs_igrab(struct erofs_inode *inode)
+{
+	++inode->i_count;
+	return inode;
+}
+
+u32 erofs_new_encode_dev(dev_t dev);
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+				   const char *name);
+int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_init_empty_dir(struct erofs_inode *dir);
+struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
 struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name);
 
include/erofs/internal.h +6 -1
index 370cfac..8a6f2cf 100644
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <stdio.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX        4096    /* # chars in a path name including nul */
@@ -172,13 +173,17 @@ struct erofs_inode {
 	} u;
 
 	char *i_srcpath;
-
+	union {
+		char *i_link;
+		FILE *i_tmpfile;
+	};
 	unsigned char datalayout;
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
 	bool compressed_idata;
 	bool lazy_tailblock;
+	bool with_tmpfile;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
include/erofs/tar.h new file +29 -0
new file mode 100644
index 0000000..268c57b
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_TAR_H
+#define __EROFS_TAR_H
+
+#include <sys/stat.h>
+
+struct erofs_pax_header {
+	struct stat st;
+	bool use_mtime;
+	bool use_size;
+	bool use_uid;
+	bool use_gid;
+	char *path, *link;
+};
+
+struct erofs_tarfile {
+	struct erofs_pax_header global;
+
+	int fd;
+	u64 offset;
+	bool index_mode, aufs;
+};
+
+int tarerofs_init_empty_dir(struct erofs_inode *inode);
+int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar);
+int tarerofs_reserve_devtable(unsigned int devices);
+int tarerofs_write_devtable(struct erofs_tarfile *tar);
+
+#endif
include/erofs/xattr.h +4 -0
index 14fc081..27e14bf 100644
@@ -72,6 +72,7 @@ static inline unsigned int xattrblock_offset(unsigned int xattr_id)
 #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
 #endif
 
+int erofs_scan_file_xattrs(struct erofs_inode *inode);
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
 int erofs_build_shared_xattrs_from_path(const char *path);
@@ -80,6 +81,9 @@ int erofs_xattr_insert_name_prefix(const char *prefix);
 void erofs_xattr_cleanup_name_prefixes(void);
 int erofs_xattr_write_name_prefixes(FILE *f);
 
+int erofs_setxattr(struct erofs_inode *inode, char *key,
+		   const void *value, size_t size);
+
 #ifdef __cplusplus
 }
 #endif
lib/Makefile.am +2 -1
index faa7311..f01cdc7 100644
@@ -19,6 +19,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/io.h \
       $(top_srcdir)/include/erofs/list.h \
       $(top_srcdir)/include/erofs/print.h \
+      $(top_srcdir)/include/erofs/tar.h \
       $(top_srcdir)/include/erofs/trace.h \
       $(top_srcdir)/include/erofs/xattr.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
@@ -29,7 +30,7 @@ noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
-		      fragments.c rb_tree.c dedupe.c
+		      fragments.c rb_tree.c dedupe.c tar.c
 
 liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/blobchunk.c +32 -15

101 lines not on this page · raw message

lib/inode.c +146 -47

291 lines not on this page · raw message

lib/tar.c new file +803 -0

803 lines not on this page · raw message

lib/xattr.c +41 -5

66 lines not on this page · raw message

mkfs/main.c +98 -36

205 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
#2 lore raw

Re: [PATCH 2/2] erofs-utils: introduce tarerofs

Quoted 21 lines · Jingbo Xu

From: Gao Xiang <[email protected]>

Let's try to add a new mode "tarerofs" for mkfs.erofs.

It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into

    a full EROFS image [--tar=f];

- Generate an EROFS manifest image to reuse tar data [--tar=i], which also enables EROFS 512-byte blocks.

The second use case is mainly prepared for OCI direct mount without OCI blob unpacking. This also adds another --aufs option to transform aufs special files into overlayfs metadata.

[ Note that --tar=f generates lots of temporary files for now which
can impact performance since the original tar stream(s) may be non-seekable. ]

Quoted 9 lines

---
change log:
1. don't return prematurely for special device files (in erofs_mkfs_build_tree()). Otherwise erofs_mkfs_build_tree() may mistakenly return a positive integer number returned from erofs_prepare_xattr_ibody(), and mistakenly make it as the pointer to the root inode, which will cause mkfs.erofs segmentation fault.
2. automatically create parent directory if it's not included as an entry in the tarball stream. (refer to tarerofs_init_default_dir())

struct erofs_dentry *tarerofs_mkdir(struct erofs_inode *dir, const char *s)

{
	struct inode *inode;

	inode = erofs_new_inode();
	if (IS_ERR(inode))
		return inode;

	inode->i_mode = S_IFDIR | 0755;
	inode->i_parent = dir;
	inode->i_uid = getuid();
	inode->i_gid = getgid();
	inode->i_mtime = sbi.build_time;
	inode->i_mtime_nsec = sbi.build_time_nsec;
	tarerofs_init_empty_dir(inode);

	d = erofs_d_alloc(pwd, s);
	d->type = EROFS_FT_DIR;
	d->inode = inode;
	return d;

}
?

Otherwise it looks good to me.

Thanks,
Gao Xiang