erofs-utils: refine blobchunk implementation
PATCH 1/2 erofs-utils: refine blobchunk implementation
- record device_id in erofs_blobchunk for later tarerofs;
- fix up endianness conversion;
- blob (hash) entries should be released in the blobchunk submodule.
--- include/erofs/hashmap.h | 5 ++--- lib/blobchunk.c | 44 ++++++++++++++++++++++++++--------------- lib/hashmap.c | 32 +++++++++++++----------------- 3 files changed, 44 insertions(+), 37 deletions(-)
include/erofs/hashmap.h +2 -3
@@ -61,7 +61,7 @@ struct hashmap_iter {/* hashmap functions */ void hashmap_init (struct hashmap *map,hashmap_cmp_fn equals_function,size_t initial_size); -void hashmap_free (struct hashmap *map,int free_entries); +int hashmap_free (struct hashmap *map);/* hashmap_entry functions */ static inline void hashmap_entry_init (void *entry,unsigned int hash)
@@ -75,8 +75,7 @@ static inline void hashmap_entry_init(void *entry, unsigned int hash)void *hashmap_get (const struct hashmap *map,const void *key,const void *keydata);void *hashmap_get_next (const struct hashmap *map,const void *entry);void hashmap_add (struct hashmap *map,void *entry); -void *hashmap_put (struct hashmap *map,void *entry); -void *hashmap_remove (struct hashmap *map,const void *key,const void *keydata); +void *hashmap_remove (struct hashmap *map,const void *key); static inline void *hashmap_get_from_hash(const struct hashmap *map, unsigned int hash,
lib/blobchunk.c +28 -16
96 lines not on this page · raw message
lib/hashmap.c +14 -18
@@ -149,20 +149,21 @@ void hashmap_init(struct hashmap *map, hashmap_cmp_fn equals_function,alloc_table (map, size); } -void hashmap_free (struct hashmap *map,int free_entries) +int hashmap_free (struct hashmap *map) { -if (!map || !map->table) -return ; -if (free_entries) { +if (map && map->table) {struct hashmap_iter iter;struct hashmap_entry *e;hashmap_iter_init (map, &iter); -while ((e =hashmap_iter_next (&iter))) -free (e); + e =hashmap_iter_next (&iter); +if (e) +return -EBUSY ; + +free (map->table); +memset (map,0 ,sizeof (*map)); } -free (map->table); -memset (map,0 ,sizeof (*map)); +return 0 ; }void *hashmap_get (const struct hashmap *map,const void *key,const void *keydata)
@@ -194,10 +195,13 @@ void hashmap_add(struct hashmap *map, void *entry)rehash (map, map->tablesize <<HASHMAP_RESIZE_BITS ); } -void *hashmap_remove (struct hashmap *map,const void *key,const void *keydata) +void *hashmap_remove (struct hashmap *map,const void *entry) {struct hashmap_entry *old; -struct hashmap_entry **e =find_entry_ptr (map, key, keydata); +struct hashmap_entry **e = &map->table[bucket (map, entry)]; + +while (*e && *e != entry) + e = &(*e)->next;if (!*e)return NULL ;
@@ -214,14 +218,6 @@ void *hashmap_remove(struct hashmap *map, const void *key, const void *keydata)return old; } -void *hashmap_put (struct hashmap *map,void *entry) -{ -struct hashmap_entry *old =hashmap_remove (map, entry,NULL ); - -hashmap_add (map, entry); -return old; -} -void hashmap_iter_init (struct hashmap *map,struct hashmap_iter *iter) { iter->map = map;
Signature · 1 line
2.24.4
PATCH v6 2/2 erofs-utils: introduce tarerofs
Let's try to add a new mode "tarerofs" for mkfs.erofs.
It mainly aims at two use cases:
- Convert a tarball (or later tarballs with a merged view) into a full EROFS image [--tar=f];
- Generate an EROFS manifest image to reuse tar data [--tar=i], which also enables EROFS 512-byte blocks.
The second use case is mainly prepared for OCI direct mount without OCI blob unpacking. This also adds another --aufs option to transform aufs special files into overlayfs metadata.
[ Note that --tar=f generates lots of temporary files for now which
can impact performance since the original tar stream(s) may be non-seekable. ]
---
changes since v5:
- Support --aufs option to convert aufs to overlayfs;
- Fix hardlink corrupted bugs;
configure.ac | 1 + include/erofs/blobchunk.h | 4 +- include/erofs/inode.h | 11 + include/erofs/internal.h | 7 +- include/erofs/tar.h | 29 ++ include/erofs/xattr.h | 4 + lib/Makefile.am | 3 +- lib/blobchunk.c | 47 ++- lib/inode.c | 191 +++++++--- lib/tar.c | 778 ++++++++++++++++++++++++++++++++++++++ lib/xattr.c | 46 ++- mkfs/main.c | 134 +++++-- 12 files changed, 1149 insertions(+), 106 deletions(-) create mode 100644 include/erofs/tar.h create mode 100644 lib/tar.c
configure.ac +1 -0
@@ -124,6 +124,7 @@ AC_CHECK_HEADERS(m4_flatten([ fcntl.h getopt.h inttypes.h + linux/aufs_type.h linux/falloc.h linux/fs.h linux/types.h
include/erofs/blobchunk.h +3 -1
@@ -14,8 +14,10 @@ extern "C"#include "erofs/internal.h" +struct erofs_blobchunk *erofs_get_unhashed_chunk (erofs_off_t chunksize, +unsigned int device_id,erofs_blk_t blkaddr);int erofs_blob_write_chunk_indexes (struct erofs_inode *inode,erofs_off_t off); -int erofs_blob_write_chunked_file (struct erofs_inode *inode); +int erofs_blob_write_chunked_file (struct erofs_inode *inode,int fd);int erofs_blob_remap (void );void erofs_blob_exit (void );int erofs_blob_init (const char *blobfile_path);
include/erofs/inode.h +11 -0
@@ -15,11 +15,22 @@ extern "C"#include "erofs/internal.h" +static inline struct erofs_inode *erofs_igrab (struct erofs_inode *inode) +{ + ++inode->i_count; +return inode; +} +unsigned char erofs_mode_to_ftype (umode_t mode);unsigned char erofs_ftype_to_dtype (unsigned int filetype);void erofs_inode_manager_init (void );unsigned int erofs_iput (struct erofs_inode *inode);erofs_nid_t erofs_lookupnid (struct erofs_inode *inode); +struct erofs_dentry *erofs_d_alloc (struct erofs_inode *parent, +const char *name); +int tarerofs_dump_tree (struct erofs_inode *dir); +int erofs_init_empty_dir (struct erofs_inode *dir); +struct erofs_inode *erofs_new_inode (void );struct erofs_inode *erofs_mkfs_build_tree_from_path (const char *path);struct erofs_inode *erofs_mkfs_build_special_from_fd (int fd,const char *name);
include/erofs/internal.h +6 -1
@@ -22,6 +22,7 @@ typedef unsigned short umode_t;#include "erofs_fs.h" #include <fcntl.h> #include <sys/types.h> /* for off_t definition */ +#include <stdio.h> #ifndef PATH_MAX #define PATH_MAX 4096/* # chars in a path name including nul */
@@ -178,13 +179,17 @@ struct erofs_inode { } u;char *i_srcpath; - +union { +char *i_link; +FILE *i_tmpfile; + };unsigned char datalayout;unsigned char inode_isize;/* inline tail-end packing size */ unsigned short idata_size;bool compressed_idata;bool lazy_tailblock; +bool with_tmpfile;unsigned int xattr_isize;unsigned int extent_isize;
include/erofs/tar.h new file +29 -0
@@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_TAR_H +#define __EROFS_TAR_H + +#include <sys/stat.h> + +struct erofs_pax_header { +struct stat st; +bool use_mtime; +bool use_size; +bool use_uid; +bool use_gid; +char *path, *link; +}; + +struct erofs_tarfile { +struct erofs_pax_header global; + +int fd; +u64 offset; +bool index_mode, aufs; +}; + +int tarerofs_init_empty_dir (struct erofs_inode *inode); +int tarerofs_parse_tar (struct erofs_inode *root,struct erofs_tarfile *tar); +int tarerofs_reserve_devtable (unsigned int devices); +int tarerofs_write_devtable (struct erofs_tarfile *tar); + +#endif
include/erofs/xattr.h +4 -0
@@ -72,6 +72,7 @@ static inline unsigned int xattrblock_offset(unsigned int xattr_id)#define XATTR_NAME_POSIX_ACL_DEFAULT "system.posix_acl_default"#endif +int erofs_scan_file_xattrs (struct erofs_inode *inode);int erofs_prepare_xattr_ibody (struct erofs_inode *inode);char *erofs_export_xattr_ibody (struct list_head *ixattrs,unsigned int size);int erofs_build_shared_xattrs_from_path (const char *path);
@@ -80,6 +81,9 @@ int erofs_xattr_insert_name_prefix(const char *prefix);void erofs_xattr_cleanup_name_prefixes (void );int erofs_xattr_write_name_prefixes (FILE *f); +int erofs_setxattr (struct erofs_inode *inode,char *key, +const void *value,size_t size); + #ifdef __cplusplus }#endif
lib/Makefile.am +2 -1
@@ -19,6 +19,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/io.h \ $(top_srcdir)/include/erofs/list.h \ $(top_srcdir)/include/erofs/print.h \ + $(top_srcdir)/include/erofs/tar.h \ $(top_srcdir)/include/erofs/trace.h \ $(top_srcdir)/include/erofs/xattr.h \ $(top_srcdir)/include/erofs/compress_hints.h \
@@ -29,7 +30,7 @@ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \ - fragments.c rb_tree.c dedupe.c + fragments.c rb_tree.c dedupe.c tar.c liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4
lib/blobchunk.c +32 -15
101 lines not on this page · raw message
lib/inode.c +144 -47
282 lines not on this page · raw message
lib/tar.c new file +778 -0
778 lines not on this page · raw message
lib/xattr.c +41 -5
66 lines not on this page · raw message
mkfs/main.c +98 -36
205 lines not on this page · raw message
Signature · 1 line
2.24.4