Skip to contentSkip to conversation

erofs-utils: mkfs: don't clean cached xattr items prematurely

PATCH In review Jingbo Xu 2 participants 4 messages Jul 5, 2023 Library code

lore
Share & tools
lore

Jingbo Xu

author
#0 lore raw

PATCH 1/1 erofs-utils: mkfs: don't clean cached xattr items prematurely

Extended attributes read from file are cached in a hash table when building shared extended attributes. However the cached xattr items for inline xattrs are cleaned up from the hash table when the processing for shared extended attributes has finished, while later the hash table is reconstructed from scratch when building the inode tree (see erofs_mkfs_build_tree_from_path()).

Don't clean up the xattr hash table halfway until mkfs exits. Also move the logic of cleaning long xattr name prefixes into erofs_cleanxattrs().

---
 include/erofs/xattr.h |  2 +-
 lib/xattr.c           | 30 ++++++++----------------------
 mkfs/main.c           |  2 +-
 3 files changed, 10 insertions(+), 24 deletions(-)
include/erofs/xattr.h +1 -1
index 14fc081..b202f78 100644
@@ -75,9 +75,9 @@ static inline unsigned int xattrblock_offset(unsigned int xattr_id)
 int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
 char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
 int erofs_build_shared_xattrs_from_path(const char *path);
+void erofs_cleanxattrs(void);
 
 int erofs_xattr_insert_name_prefix(const char *prefix);
-void erofs_xattr_cleanup_name_prefixes(void);
 int erofs_xattr_write_name_prefixes(FILE *f);
 
 #ifdef __cplusplus
lib/xattr.c +8 -22
index 7d7dc54..8d0079f 100644
@@ -547,24 +547,23 @@ fail:
 	return ret;
 }
 
-static void erofs_cleanxattrs(bool sharedxattrs)
+void erofs_cleanxattrs(void)
 {
 	unsigned int i;
 	struct xattr_item *item;
 	struct hlist_node *tmp;
+	struct ea_type_node *tnode, *n;
 
 	hash_for_each_safe(ea_hashtable, i, tmp, item, node) {
-		if (sharedxattrs && item->shared_xattr_id >= 0)
-			continue;
-
 		hash_del(&item->node);
 		free(item);
 	}
 
-	if (sharedxattrs)
-		return;
-
-	shared_xattrs_count = 0;
+	list_for_each_entry_safe(tnode, n, &ea_name_prefixes, list) {
+		list_del(&tnode->list);
+		free((void *)tnode->type.prefix);
+		free(tnode);
+	}
 }
 
 static int comp_shared_xattr_item(const void *a, const void *b)
@@ -676,7 +675,7 @@ int erofs_build_shared_xattrs_from_path(const char *path)
 		return ret;
 
 	if (!shared_xattrs_count)
-		goto out;
+		return 0;
 
 	sorted_n = malloc((shared_xattrs_count + 1) * sizeof(n));
 	if (!sorted_n)
@@ -735,8 +734,6 @@ int erofs_build_shared_xattrs_from_path(const char *path)
 	ret = dev_write(buf, erofs_btell(bh, false), shared_xattrs_size);
 	free(buf);
 	erofs_bdrop(bh, false);
-out:
-	erofs_cleanxattrs(true);
 	return ret;
 }
 
@@ -1340,14 +1337,3 @@ int erofs_xattr_insert_name_prefix(const char *prefix)
 	list_add_tail(&tnode->list, &ea_name_prefixes);
 	return 0;
 }
-
-void erofs_xattr_cleanup_name_prefixes(void)
-{
-	struct ea_type_node *tnode, *n;
-
-	list_for_each_entry_safe(tnode, n, &ea_name_prefixes, list) {
-		list_del(&tnode->list);
-		free((void *)tnode->type.prefix);
-		free(tnode);
-	}
-}
mkfs/main.c +1 -1
index ac208e5..390ef17 100644
@@ -942,7 +942,7 @@ exit:
 	if (cfg.c_fragments)
 		z_erofs_fragments_exit();
 	erofs_packedfile_exit();
-	erofs_xattr_cleanup_name_prefixes();
+	erofs_cleanxattrs();
 	erofs_exit_configure();
 
 	if (err) {
Signature · 1 line
2.19.1.6.gb485710b
#1 lore raw
Quoted 41 lines · Jingbo Xu

Extended attributes read from file are cached in a hash table when building shared extended attributes. However the cached xattr items for inline xattrs are cleaned up from the hash table when the processing for shared extended attributes has finished, while later the hash table is reconstructed from scratch when building the inode tree (see erofs_mkfs_build_tree_from_path()).

Don't clean up the xattr hash table halfway until mkfs exits. Also move the logic of cleaning long xattr name prefixes into erofs_cleanxattrs().

---
  include/erofs/xattr.h |  2 +-
  lib/xattr.c           | 30 ++++++++----------------------
  mkfs/main.c           |  2 +-
  3 files changed, 10 insertions(+), 24 deletions(-)
include/erofs/xattr.h +1 -1
index 14fc081..b202f78 100644
@@ -75,9 +75,9 @@ static inline unsigned int xattrblock_offset(unsigned int xattr_id)
  int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
  char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
  int erofs_build_shared_xattrs_from_path(const char *path);
+void erofs_cleanxattrs(void);
 
  int erofs_xattr_insert_name_prefix(const char *prefix);
-void erofs_xattr_cleanup_name_prefixes(void);
  int erofs_xattr_write_name_prefixes(FILE *f);
 
  #ifdef __cplusplus
lib/xattr.c +0 -1
index 7d7dc54..8d0079f 100644
@@ -547,24 +547,23 @@ fail:
  	return ret;
  }
 
-static void erofs_cleanxattrs(bool sharedxattrs)

lib/xattr.c fail: · line 558

Quoted 10 lines
lib/xattr.c +2 -2
+void erofs_cleanxattrs(void)
  {
  	unsigned int i;
  	struct xattr_item *item;
  	struct hlist_node *tmp;
+	struct ea_type_node *tnode, *n;
 
  	hash_for_each_safe(ea_hashtable, i, tmp, item, node) {
-		if (sharedxattrs && item->shared_xattr_id >= 0)
-			continue;

I'm not sure it's the expected behavior. Previously we will remove all non-shared xattrs which are below xattr threshold in the shared xattr generation process.

But now, such non-shared xattrs will be left in memory. What the benefits of this? In addition, I'm not sure if we need to add non-shared (inline) xattrs to hash table as well.

Thanks,
Gao Xiang

author
#2 lore raw
Quoted 48 lines · Gao Xiang
Quoted 45 lines · Jingbo Xu

Extended attributes read from file are cached in a hash table when building shared extended attributes.  However the cached xattr items for inline xattrs are cleaned up from the hash table when the processing for shared extended attributes has finished, while later the hash table is reconstructed from scratch when building the inode tree (see erofs_mkfs_build_tree_from_path()).

Don't clean up the xattr hash table halfway until mkfs exits.  Also move the logic of cleaning long xattr name prefixes into erofs_cleanxattrs().

---
  include/erofs/xattr.h |  2 +-
  lib/xattr.c           | 30 ++++++++---------------------- mkfs/main.c           |  2 +-

  3 files changed, 10 insertions(+), 24 deletions(-)
include/erofs/xattr.h +0 -0
index 14fc081..b202f78 100644
@@ -75,9 +75,9 @@ static inline unsigned int
 xattrblock_offset(unsigned int xattr_id)
   int erofs_prepare_xattr_ibody(struct erofs_inode *inode);

  char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size);
  int erofs_build_shared_xattrs_from_path(const char *path); +void erofs_cleanxattrs(void);
    int erofs_xattr_insert_name_prefix(const char *prefix); -void erofs_xattr_cleanup_name_prefixes(void);
  int erofs_xattr_write_name_prefixes(FILE *f);
    #ifdef __cplusplus

lib/xattr.c +1 -0
index 7d7dc54..8d0079f 100644
@@ -547,24 +547,23 @@ fail:
       return ret;
   }
   -static void erofs_cleanxattrs(bool sharedxattrs)
+void erofs_cleanxattrs(void)
   {
       unsigned int i;
       struct xattr_item *item;
       struct hlist_node *tmp;
Quoted 8 lines

lib/xattr.c fail: · line 558

Quoted 4 lines
lib/xattr.c +1 -2
+    struct ea_type_node *tnode, *n;
         hash_for_each_safe(ea_hashtable, i, tmp, item, node) {
-        if (sharedxattrs && item->shared_xattr_id >= 0)
-            continue;

I'm not sure it's the expected behavior. Previously we will remove all non-shared xattrs which are below xattr threshold in the shared xattr generation process.

What's the purpose of removing these non-shared xattrs while leaving shared xattrs there?

Quoted 4 lines

But now, such non-shared xattrs will be left in memory.  What the benefits of this?  In addition, I'm not sure if we need to add non-shared (inline) xattrs to hash table as well.

Later erofs_mkfs_build_tree_from_path() needs to reconstruct these non-shared xattrs, which have been previously removed.

erofs_mkfs_build_tree_from_path
erofs_mkfs_build_tree
erofs_prepare_xattr_ibody
read_xattrs_from_file

Signature · 2 lines
Thanks,
Jingbo
#3 lore raw
Quoted 60 lines · Jingbo Xu
Quoted 48 lines · Gao Xiang
Quoted 45 lines · Jingbo Xu

Extended attributes read from file are cached in a hash table when building shared extended attributes.  However the cached xattr items for inline xattrs are cleaned up from the hash table when the processing for shared extended attributes has finished, while later the hash table is reconstructed from scratch when building the inode tree (see erofs_mkfs_build_tree_from_path()).

Don't clean up the xattr hash table halfway until mkfs exits.  Also move the logic of cleaning long xattr name prefixes into erofs_cleanxattrs().

---
   include/erofs/xattr.h |  2 +-
   lib/xattr.c           | 30 ++++++++----------------------
   mkfs/main.c           |  2 +-
   3 files changed, 10 insertions(+), 24 deletions(-)
include/erofs/xattr.h +1 -1
index 14fc081..b202f78 100644
@@ -75,9 +75,9 @@ static inline unsigned int
 xattrblock_offset(unsigned int xattr_id)
   int erofs_prepare_xattr_ibody(struct erofs_inode *inode);
   char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned
 int size);
   int erofs_build_shared_xattrs_from_path(const char *path);
+void erofs_cleanxattrs(void);
     int erofs_xattr_insert_name_prefix(const char *prefix);
-void erofs_xattr_cleanup_name_prefixes(void);
   int erofs_xattr_write_name_prefixes(FILE *f);
     #ifdef __cplusplus
lib/xattr.c +1 -0
index 7d7dc54..8d0079f 100644
@@ -547,24 +547,23 @@ fail:
       return ret;
   }
   -static void erofs_cleanxattrs(bool sharedxattrs)
+void erofs_cleanxattrs(void)
   {
       unsigned int i;
       struct xattr_item *item;
       struct hlist_node *tmp;
Quoted 8 lines

lib/xattr.c fail: · line 558

Quoted 4 lines
lib/xattr.c +1 -2
+    struct ea_type_node *tnode, *n;
         hash_for_each_safe(ea_hashtable, i, tmp, item, node) {
-        if (sharedxattrs && item->shared_xattr_id >= 0)
-            continue;

I'm not sure it's the expected behavior. Previously we will remove all non-shared xattrs which are below xattr threshold in the shared xattr generation process.

Quoted 2 lines

What's the purpose of removing these non-shared xattrs while leaving shared xattrs there?

shared xattrs will be used for later tree walking. non shared xattrs are not.

Quoted 6 lines
Quoted 5 lines

But now, such non-shared xattrs will be left in memory.  What the benefits of this?  In addition, I'm not sure if we need to add non-shared (inline) xattrs to hash table as well.

Quoted 7 lines

Later erofs_mkfs_build_tree_from_path() needs to reconstruct these non-shared xattrs, which have been previously removed.

erofs_mkfs_build_tree_from_path
erofs_mkfs_build_tree
erofs_prepare_xattr_ibody
read_xattrs_from_file

That is true, but in principle these are seperate two stuffs:

- erofs_build_shared_xattrs_from_path generates shared xattrs from a tree.
- erofs_mkfs_build_tree_from_path generate trees with the previous shared xattrs.

These two paths can even be different if we'd like to build a wide shared xattr array.

For example, you could build shared xattrs from "/usr" but only pack "/usr/lib" then.

Thanks,
Gao Xiang

Quoted 2 lines