Skip to contentSkip to conversation

erofs-utils: refine blobchunk implementation

PATCH New Gao Xiang 1 participant 2 messages Apr 21, 2023 Library code

lore
Share & tools
lore

Gao Xiang

author
#0 lore raw

PATCH 1/2 erofs-utils: refine blobchunk implementation

- record device_id in erofs_blobchunk for later tarerofs;

- fix up endianness conversion;

- blob (hash) entries should be released in the blobchunk submodule.

---
 include/erofs/hashmap.h |  5 ++---
 lib/blobchunk.c         | 44 ++++++++++++++++++++++++++---------------
 lib/hashmap.c           | 32 +++++++++++++-----------------
 3 files changed, 44 insertions(+), 37 deletions(-)
include/erofs/hashmap.h +2 -3
index 3d38578..d25092d 100644
@@ -61,7 +61,7 @@ struct hashmap_iter {
 /* hashmap functions */
 void hashmap_init(struct hashmap *map, hashmap_cmp_fn equals_function,
 		  size_t initial_size);
-void hashmap_free(struct hashmap *map, int free_entries);
+int hashmap_free(struct hashmap *map);
 
 /* hashmap_entry functions */
 static inline void hashmap_entry_init(void *entry, unsigned int hash)
@@ -75,8 +75,7 @@ static inline void hashmap_entry_init(void *entry, unsigned int hash)
 void *hashmap_get(const struct hashmap *map, const void *key, const void *keydata);
 void *hashmap_get_next(const struct hashmap *map, const void *entry);
 void hashmap_add(struct hashmap *map, void *entry);
-void *hashmap_put(struct hashmap *map, void *entry);
-void *hashmap_remove(struct hashmap *map, const void *key, const void *keydata);
+void *hashmap_remove(struct hashmap *map, const void *key);
 
 static inline void *hashmap_get_from_hash(const struct hashmap *map,
 					  unsigned int hash,
lib/blobchunk.c +28 -16

96 lines not on this page · raw message

lib/hashmap.c +14 -18
index e11bd8d..45916ae 100644
@@ -149,20 +149,21 @@ void hashmap_init(struct hashmap *map, hashmap_cmp_fn equals_function,
 	alloc_table(map, size);
 }
 
-void hashmap_free(struct hashmap *map, int free_entries)
+int hashmap_free(struct hashmap *map)
 {
-	if (!map || !map->table)
-		return;
-	if (free_entries) {
+	if (map && map->table) {
 		struct hashmap_iter iter;
 		struct hashmap_entry *e;
 
 		hashmap_iter_init(map, &iter);
-		while ((e = hashmap_iter_next(&iter)))
-			free(e);
+		e = hashmap_iter_next(&iter);
+		if (e)
+			return -EBUSY;
+
+		free(map->table);
+		memset(map, 0, sizeof(*map));
 	}
-	free(map->table);
-	memset(map, 0, sizeof(*map));
+	return 0;
 }
 
 void *hashmap_get(const struct hashmap *map, const void *key, const void *keydata)
@@ -194,10 +195,13 @@ void hashmap_add(struct hashmap *map, void *entry)
 		rehash(map, map->tablesize << HASHMAP_RESIZE_BITS);
 }
 
-void *hashmap_remove(struct hashmap *map, const void *key, const void *keydata)
+void *hashmap_remove(struct hashmap *map, const void *entry)
 {
 	struct hashmap_entry *old;
-	struct hashmap_entry **e = find_entry_ptr(map, key, keydata);
+	struct hashmap_entry **e = &map->table[bucket(map, entry)];
+
+	while (*e && *e != entry)
+		e = &(*e)->next;
 
 	if (!*e)
 		return NULL;
@@ -214,14 +218,6 @@ void *hashmap_remove(struct hashmap *map, const void *key, const void *keydata)
 	return old;
 }
 
-void *hashmap_put(struct hashmap *map, void *entry)
-{
-	struct hashmap_entry *old = hashmap_remove(map, entry, NULL);
-
-	hashmap_add(map, entry);
-	return old;
-}
-
 void hashmap_iter_init(struct hashmap *map, struct hashmap_iter *iter)
 {
 	iter->map = map;
Signature · 1 line
2.24.4
author
#1 lore raw

PATCH v6 2/2 erofs-utils: introduce tarerofs

Let's try to add a new mode "tarerofs" for mkfs.erofs.

It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full EROFS image [--tar=f];

- Generate an EROFS manifest image to reuse tar data [--tar=i], which also enables EROFS 512-byte blocks.

The second use case is mainly prepared for OCI direct mount without OCI blob unpacking. This also adds another --aufs option to transform aufs special files into overlayfs metadata.

[ Note that --tar=f generates lots of temporary files for now which
can impact performance since the original tar stream(s) may be non-seekable. ]

---
changes since v5:
- Support --aufs option to convert aufs to overlayfs;
- Fix hardlink corrupted bugs;

 configure.ac              |   1 +
 include/erofs/blobchunk.h |   4 +-
 include/erofs/inode.h     |  11 +
 include/erofs/internal.h  |   7 +-
 include/erofs/tar.h       |  29 ++
 include/erofs/xattr.h     |   4 +
 lib/Makefile.am           |   3 +-
 lib/blobchunk.c           |  47 ++-
 lib/inode.c               | 191 +++++++---
 lib/tar.c                 | 778 ++++++++++++++++++++++++++++++++++++++
 lib/xattr.c               |  46 ++-
 mkfs/main.c               | 134 +++++--
 12 files changed, 1149 insertions(+), 106 deletions(-)
 create mode 100644 include/erofs/tar.h
 create mode 100644 lib/tar.c
configure.ac +1 -0
index 4dbe86f..0f586dd 100644
@@ -124,6 +124,7 @@ AC_CHECK_HEADERS(m4_flatten([
 	fcntl.h
 	getopt.h
 	inttypes.h
+	linux/aufs_type.h
 	linux/falloc.h
 	linux/fs.h
 	linux/types.h
include/erofs/blobchunk.h +3 -1
index 49cb7bf..4269d82 100644
@@ -14,8 +14,10 @@ extern "C"
 
 #include "erofs/internal.h"
 
+struct erofs_blobchunk *erofs_get_unhashed_chunk(erofs_off_t chunksize,
+			unsigned int device_id, erofs_blk_t blkaddr);
 int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
-int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode, int fd);
 int erofs_blob_remap(void);
 void erofs_blob_exit(void);
 int erofs_blob_init(const char *blobfile_path);
include/erofs/inode.h +11 -0
index 058a235..1c3e3cb 100644
@@ -15,11 +15,22 @@ extern "C"
 
 #include "erofs/internal.h"
 
+static inline struct erofs_inode *erofs_igrab(struct erofs_inode *inode)
+{
+	++inode->i_count;
+	return inode;
+}
+
 unsigned char erofs_mode_to_ftype(umode_t mode);
 unsigned char erofs_ftype_to_dtype(unsigned int filetype);
 void erofs_inode_manager_init(void);
 unsigned int erofs_iput(struct erofs_inode *inode);
 erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+				   const char *name);
+int tarerofs_dump_tree(struct erofs_inode *dir);
+int erofs_init_empty_dir(struct erofs_inode *dir);
+struct erofs_inode *erofs_new_inode(void);
 struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
 struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name);
 
include/erofs/internal.h +6 -1
index b3d04be..7171fc3 100644
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;
 #include "erofs_fs.h"
 #include <fcntl.h>
 #include <sys/types.h> /* for off_t definition */
+#include <stdio.h>
 
 #ifndef PATH_MAX
 #define PATH_MAX        4096    /* # chars in a path name including nul */
@@ -178,13 +179,17 @@ struct erofs_inode {
 	} u;
 
 	char *i_srcpath;
-
+	union {
+		char *i_link;
+		FILE *i_tmpfile;
+	};
 	unsigned char datalayout;
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
 	bool compressed_idata;
 	bool lazy_tailblock;
+	bool with_tmpfile;
 
 	unsigned int xattr_isize;
 	unsigned int extent_isize;
include/erofs/tar.h new file +29 -0
new file mode 100644
index 0000000..268c57b
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+#ifndef __EROFS_TAR_H
+#define __EROFS_TAR_H
+
+#include <sys/stat.h>
+
+struct erofs_pax_header {
+	struct stat st;
+	bool use_mtime;
+	bool use_size;
+	bool use_uid;
+	bool use_gid;
+	char *path, *link;
+};
+
+struct erofs_tarfile {
+	struct erofs_pax_header global;
+
+	int fd;
+	u64 offset;
+	bool index_mode, aufs;
+};
+
+int tarerofs_init_empty_dir(struct erofs_inode *inode);
+int tarerofs_parse_tar(struct erofs_inode *root, struct erofs_tarfile *tar);
+int tarerofs_reserve_devtable(unsigned int devices);
+int tarerofs_write_devtable(struct erofs_tarfile *tar);
+
+#endif
include/erofs/xattr.h +4 -0
index 14fc081..27e14bf 100644
@@ -72,6 +72,7 @@ static inline unsigned int xattrblock_offset(unsigned int xattr_id)
 #define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"
 #endif
 
+int erofs_scan_file_xattrs(struct erofs_inode *inode);
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
 int erofs_build_shared_xattrs_from_path(const char *path);
@@ -80,6 +81,9 @@ int erofs_xattr_insert_name_prefix(const char *prefix);
 void erofs_xattr_cleanup_name_prefixes(void);
 int erofs_xattr_write_name_prefixes(FILE *f);
 
+int erofs_setxattr(struct erofs_inode *inode, char *key,
+		   const void *value, size_t size);
+
 #ifdef __cplusplus
 }
 #endif
lib/Makefile.am +2 -1
index faa7311..f01cdc7 100644
@@ -19,6 +19,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/io.h \
       $(top_srcdir)/include/erofs/list.h \
       $(top_srcdir)/include/erofs/print.h \
+      $(top_srcdir)/include/erofs/tar.h \
       $(top_srcdir)/include/erofs/trace.h \
       $(top_srcdir)/include/erofs/xattr.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
@@ -29,7 +30,7 @@ noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
-		      fragments.c rb_tree.c dedupe.c
+		      fragments.c rb_tree.c dedupe.c tar.c
 
 liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/blobchunk.c +32 -15

101 lines not on this page · raw message

lib/inode.c +144 -47

282 lines not on this page · raw message

lib/tar.c new file +778 -0

778 lines not on this page · raw message

lib/xattr.c +41 -5

66 lines not on this page · raw message

mkfs/main.c +98 -36

205 lines not on this page · raw message

Signature · 1 line
2.24.4