Skip to contentSkip to conversation

erofs-utils: introduce xattr name bloom filter

PATCH Superseded Jingbo Xu 2 participants 10 messages Aug 29, 2023 Library code

v6 of 7 Superseded Latest is v7 →

lore
Share & tools
lore
author
#7 lore raw

PATCH v6 3/3 erofs-utils: mkfs: enable xattr name filter

In reply to Jingbo Xu

Introduce "-Exattr-name-filter" option to enable the xattr name bloom filter feature.

---
 include/erofs/config.h   |  1 +
 include/erofs/internal.h |  1 +
 lib/xattr.c              | 63 ++++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |  7 +++++
 4 files changed, 72 insertions(+)
include/erofs/config.h +1 -0
index 8f52d2c..c51f0cd 100644
@@ -53,6 +53,7 @@ struct erofs_configure {
 	bool c_ignore_mtime;
 	bool c_showprogress;
 	bool c_extra_ea_name_prefixes;
+	bool c_xattr_name_filter;
 
 #ifdef HAVE_LIBSELINUX
 	struct selabel_handle *sehnd;
include/erofs/internal.h +1 -0
index 3e73eef..382024a 100644
@@ -139,6 +139,7 @@ EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
 EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
 EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
 EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
+EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)
 
 #define EROFS_I_EA_INITED	(1 << 0)
 #define EROFS_I_Z_INITED	(1 << 1)
lib/xattr.c +63 -0

81 lines not on this page · raw message

mkfs/main.c +7 -0
index c03a7a8..fad80b1 100644
@@ -245,6 +245,13 @@ handle_fragment:
 				return -EINVAL;
 			cfg.c_dedupe = true;
 		}
+
+		if (MATCH_EXTENTED_OPT("xattr-name-filter", token, keylen)) {
+			if (vallen)
+				return -EINVAL;
+			cfg.c_xattr_name_filter = true;
+			erofs_sb_set_xattr_filter(&sbi);
+		}
 	}
 	return 0;
 }
Signature · 1 line
2.19.1.6.gb485710b

Jingbo Xu

author
#0 lore raw

v6 cover erofs-utils: introduce xattr name bloom filter

changes since v5:
- patch 1: update distribution license
- patch 3: rebase to the latest -dev branch

changes since v4:
- patch 3: the feature is refactored and implemented in a more cohesive way, that is, the hashbit is calculated in erofs_export_xattr_ibody(). For xattr_item in the long xattr name prefix style, reconstruct the full xattr name first, then derive the corresponding predefined short name prefix and finally calculate the hashbit.

changes since v3:
- patch 3: "-Exattr-name-filter" option rather than "--xattr-filter" option is newly introduced to enable this feature (Gao Xiang)

changes since v2:
- patch 2: introduce xattr_filter_reserved in on-disk superblock; remove EROFS_XATTR_FILTER_MASK
- patch 3: xattr_filter_reserved is always initialized to 0 by default

changes since RFC:
- the number of hash functions is 1, and now it's implemented as:
xxh32(name, strlen(name), EROFS_XATTR_FILTER_SEED + index),
where the constant magic number EROFS_XATTR_FILTER_SEED [*] is used to give a better spread for the mapping. (Alexander Larsson)
- fix the value of EROFS_FEATURE_COMPAT_XATTR_FILTER; rename EROFS_XATTR_BLOOM_* to EROFS_XATTR_FILTER_* (Gao Xiang)

RFC: https://lore.kernel.org/all/[email protected]/
v2: https://lore.kernel.org/all/[email protected]/
v3: https://lore.kernel.org/all/[email protected]/
v4: https://lore.kernel.org/all/[email protected]/
v5: https://lore.kernel.org/all/[email protected]/

The xattr bloom filter feature is used to boost the negative xattr lookup.

Refer to the kernel patch set [*] for more details.

[*] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=fd73a4395d47

Jingbo Xu (3):
erofs-utils: add xxh32 library
erofs-utils: update on-disk format for xattr name filter
erofs-utils: mkfs: enable xattr name filter

 include/erofs/config.h   |   1 +
 include/erofs/internal.h |   1 +
 include/erofs/xxhash.h   |  72 ++++++++++++++++++++++
 include/erofs_fs.h       |  12 +++-
 lib/Makefile.am          |   3 +-
 lib/xattr.c              |  63 ++++++++++++++++++++
 lib/xxhash.c             | 125 +++++++++++++++++++++++++++++++++++++++
 mkfs/main.c              |   7 +++
 8 files changed, 281 insertions(+), 3 deletions(-)
 create mode 100644 include/erofs/xxhash.h
 create mode 100644 lib/xxhash.c
Signature · 1 line
2.19.1.6.gb485710b
author
#1 lore raw

PATCH v6 1/3 erofs-utils: add xxh32 library

Add xxh32 library which could be used by following xattr bloom filter feature.

---
 include/erofs/xxhash.h |  72 ++++++++++++++++++++++++
 lib/Makefile.am        |   3 +-
 lib/xxhash.c           | 125 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/xxhash.h
 create mode 100644 lib/xxhash.c
include/erofs/xxhash.h new file +72 -0
new file mode 100644
index 0000000..5459bd8
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only
+/*
+ * The xxhash is copied from the linux kernel at:
+ *	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/xxhash.c
+ *
+ * The original copyright is:
+ *
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the
+ *     distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+#ifndef __EROFS_XXHASH_H
+#define __EROFS_XXHASH_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "defs.h"
+
+/**
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
+ *
+ * @input:  The data to hash.
+ * @length: The length of the data to hash.
+ * @seed:   The seed can be used to alter the result predictably.
+ *
+ * Return:  The 32-bit hash of the data.
+ */
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +2 -1
index 7a5dc03..3e09516 100644
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/xattr.h \
       $(top_srcdir)/include/erofs/compress_hints.h \
       $(top_srcdir)/include/erofs/fragments.h \
+      $(top_srcdir)/include/erofs/xxhash.h \
       $(top_srcdir)/lib/liberofs_private.h
 
 noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
 		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
 		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c
+		      block_list.c xxhash.c
 
 liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
 if ENABLE_LZ4
lib/xxhash.c new file +125 -0

125 lines not on this page · raw message

Signature · 1 line
2.19.1.6.gb485710b
author
#4 lore raw

PATCH v6 2/3 erofs-utils: update on-disk format for xattr name filter

In reply to Jingbo Xu

The xattr name bloom filter feature is going to be introduced to speed up the negative xattr lookup, e.g. system.posix_acl_[access|default] lookup when running "ls -lR" workload.

There are some commonly used extended attributes (n) and the total number of these is approximately 30.

trusted.overlay.opaque
trusted.overlay.redirect
trusted.overlay.origin
trusted.overlay.impure
trusted.overlay.nlink
trusted.overlay.upper
trusted.overlay.metacopy
trusted.overlay.protattr
user.overlay.opaque
user.overlay.redirect
user.overlay.origin
user.overlay.impure
user.overlay.nlink
user.overlay.upper
user.overlay.metacopy
user.overlay.protattr
security.evm
security.ima
security.selinux
security.SMACK64
security.SMACK64IPIN
security.SMACK64IPOUT
security.SMACK64EXEC
security.SMACK64TRANSMUTE
security.SMACK64MMAP
security.apparmor
security.capability
system.posix_acl_access
system.posix_acl_default
user.mime_type

Given the number of bits of the bloom filter (m) is 32, the optimal value for the number of the hash functions (k) is 1 (ln2 * m/n = 0.74).

The single hash function is implemented as:

xxh32(name, strlen(name), EROFS_XATTR_FILTER_SEED + index)

where index represents the index of corresponding predefined short name prefix, while name represents the name string after stripping the above predefined name prefix.

The constant magic number EROFS_XATTR_FILTER_SEED, i.e. 0x25BBE08F, is used to give a better spread when mapping these 30 extended attributes into 32-bit bloom filter as:

	bit  0: security.ima
	bit  1:
	bit  2: trusted.overlay.nlink
	bit  3:
	bit  4: user.overlay.nlink
	bit  5: trusted.overlay.upper
	bit  6: user.overlay.origin
	bit  7: trusted.overlay.protattr
	bit  8: security.apparmor
	bit  9: user.overlay.protattr

bit 10: user.overlay.opaque
bit 11: security.selinux
bit 12: security.SMACK64TRANSMUTE
bit 13: security.SMACK64
bit 14: security.SMACK64MMAP
bit 15: user.overlay.impure
bit 16: security.SMACK64IPIN
bit 17: trusted.overlay.redirect
bit 18: trusted.overlay.origin
bit 19: security.SMACK64IPOUT
bit 20: trusted.overlay.opaque
bit 21: system.posix_acl_default
bit 22:
bit 23: user.mime_type
bit 24: trusted.overlay.impure
bit 25: security.SMACK64EXEC
bit 26: user.overlay.redirect
bit 27: user.overlay.upper
bit 28: security.evm
bit 29: security.capability
bit 30: system.posix_acl_access
bit 31: trusted.overlay.metacopy, user.overlay.metacopy

h_name_filter is introduced to the on-disk per-inode xattr header to place the corresponding xattr name filter, where bit value 1 indicates non-existence for compatibility.

This feature is indicated by EROFS_FEATURE_COMPAT_XATTR_FILTER compatible feature bit.

Reserve one byte in on-disk superblock as the on-disk format for xattr name filter may change in the future. With this flag we don't need bothering these compatible bits again at that time.

---
 include/erofs_fs.h | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
include/erofs_fs.h +10 -2
index 850438a..2a111a4 100644
@@ -14,6 +14,7 @@
 
 #define EROFS_FEATURE_COMPAT_SB_CHKSUM          0x00000001
 #define EROFS_FEATURE_COMPAT_MTIME              0x00000002
+#define EROFS_FEATURE_COMPAT_XATTR_FILTER	0x00000004
 
 /*
  * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should
@@ -82,7 +83,8 @@ struct erofs_super_block {
 	__u8 xattr_prefix_count;	/* # of long xattr name prefixes */
 	__le32 xattr_prefix_start;	/* start of long xattr prefixes */
 	__le64 packed_nid;	/* nid of the special packed inode */
-	__u8 reserved2[24];
+	__u8 xattr_filter_reserved; /* reserved for xattr name filter */
+	__u8 reserved2[23];
 };
 
 /*
@@ -201,7 +203,7 @@ struct erofs_inode_extended {
  * for read-only fs, no need to introduce h_refcount
  */
 struct erofs_xattr_ibody_header {
-	__le32 h_reserved;
+	__le32 h_name_filter;		/* bit value 1 indicates not-present */
 	__u8   h_shared_count;
 	__u8   h_reserved2[7];
 	__le32 h_shared_xattrs[0];      /* shared xattr id array */
@@ -222,6 +224,12 @@ struct erofs_xattr_ibody_header {
 #define EROFS_XATTR_LONG_PREFIX		0x80
 #define EROFS_XATTR_LONG_PREFIX_MASK	0x7f
 
+#define EROFS_XATTR_NAME_LEN_MAX	UCHAR_MAX
+
+#define EROFS_XATTR_FILTER_BITS		32
+#define EROFS_XATTR_FILTER_DEFAULT	UINT32_MAX
+#define EROFS_XATTR_FILTER_SEED		0x25BBE08F
+
 /* xattr entry (for both inline & shared xattrs) */
 struct erofs_xattr_entry {
 	__u8   e_name_len;      /* length of name */
Signature · 1 line
2.19.1.6.gb485710b
#2 lore raw

Re: [PATCH v6 1/3] erofs-utils: add xxh32 library

In reply to Jingbo Xu
Quoted 18 lines · Jingbo Xu

Add xxh32 library which could be used by following xattr bloom filter feature.

---
  include/erofs/xxhash.h |  72 ++++++++++++++++++++++++
  lib/Makefile.am        |   3 +-
  lib/xxhash.c           | 125 +++++++++++++++++++++++++++++++++++++++++
  3 files changed, 199 insertions(+), 1 deletion(-)

create mode 100644 include/erofs/xxhash.h
create mode 100644 lib/xxhash.c

include/erofs/xxhash.h new file +0 -0
new file mode 100644
index 0000000..5459bd8

include/erofs/xxhash.h line 1

Quoted 1 line
include/erofs/xxhash.h new file +1 -0
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only

You should use:
/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */

for each header, but I guess we could drop the copyright below since it's just a declaration?

Quoted 44 lines
include/erofs/xxhash.h new file +44 -0
+/*
+ * The xxhash is copied from the linux kernel at:
+ *	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/xxhash.c
+ *
+ * The original copyright is:
+ *
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the
+ *     distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */

include/erofs/xxhash.h line 55

Quoted 10 lines
include/erofs/xxhash.h new file +10 -0
+
+#ifndef __EROFS_XXHASH_H
+#define __EROFS_XXHASH_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "defs.h"

#include <stdint.h> ?

Quoted 80 lines
include/erofs/xxhash.h new file +17 -0
+
+/**
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
+ *
+ * @input:  The data to hash.
+ * @length: The length of the data to hash.
+ * @seed:   The seed can be used to alter the result predictably.
+ *
+ * Return:  The 32-bit hash of the data.
+ */
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +2 -1
index 7a5dc03..3e09516 100644
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
        $(top_srcdir)/include/erofs/xattr.h \
        $(top_srcdir)/include/erofs/compress_hints.h \
        $(top_srcdir)/include/erofs/fragments.h \
+      $(top_srcdir)/include/erofs/xxhash.h \
        $(top_srcdir)/lib/liberofs_private.h
 
  noinst_HEADERS += compressor.h
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
  		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
  		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
  		      fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-		      block_list.c
+		      block_list.c xxhash.c
 
  liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
  if ENABLE_LZ4
lib/xxhash.c new file +35 -0
new file mode 100644
index 0000000..3d77fe3
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only
+/*
+ * The xxhash is copied from the linux kernel at:
+ *	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/xxhash.c
+ *
+ * The original copyright is:
+ *
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the
+ *     distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *

lib/xxhash.c line 45

Quoted 10 lines
lib/xxhash.c new file +10 -0
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */

#include "erofs/defs.h"

lib/xxhash.c line 47

Quoted 2 lines
lib/xxhash.c new file +2 -0
+#include "erofs/xxhash.h"
+

Thanks,
Gao Xiang

#5 lore raw

Re: [PATCH v6 2/3] erofs-utils: update on-disk format for xattr name filter

In reply to Jingbo Xu
Quoted 3 lines · Jingbo Xu

The xattr name bloom filter feature is going to be introduced to speed up the negative xattr lookup, e.g. system.posix_acl_[access|default] lookup when running "ls -lR" workload.

I think you could just say
"Let's keep in sync with kernel commit <12-char commit id>."

here.

...

Quoted 1 line

---

...

Quoted 1 line

+#define EROFS_XATTR_NAME_LEN_MAX UCHAR_MAX

where is EROFS_XATTR_NAME_LEN_MAX used?

Thanks,
Gao Xiang

#8 lore raw

Re: [PATCH v6 3/3] erofs-utils: mkfs: enable xattr name filter

In reply to Jingbo Xu
Quoted 92 lines · Jingbo Xu

Introduce "-Exattr-name-filter" option to enable the xattr name bloom filter feature.

---
  include/erofs/config.h   |  1 +
  include/erofs/internal.h |  1 +
  lib/xattr.c              | 63 ++++++++++++++++++++++++++++++++++++++++
  mkfs/main.c              |  7 +++++
  4 files changed, 72 insertions(+)
include/erofs/config.h +1 -0
index 8f52d2c..c51f0cd 100644
@@ -53,6 +53,7 @@ struct erofs_configure {
  	bool c_ignore_mtime;
  	bool c_showprogress;
  	bool c_extra_ea_name_prefixes;
+	bool c_xattr_name_filter;
 
  #ifdef HAVE_LIBSELINUX
  	struct selabel_handle *sehnd;
include/erofs/internal.h +1 -0
index 3e73eef..382024a 100644
@@ -139,6 +139,7 @@ EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
  EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
  EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
  EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
+EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)
 
  #define EROFS_I_EA_INITED	(1 << 0)
  #define EROFS_I_Z_INITED	(1 << 1)
lib/xattr.c +42 -0
index 46a301a..325241d 100644
@@ -18,6 +18,7 @@
  #include "erofs/cache.h"
  #include "erofs/io.h"
  #include "erofs/fragments.h"
+#include "erofs/xxhash.h"
  #include "liberofs_private.h"
 
  #define EA_HASHTABLE_BITS 16
@@ -783,6 +784,63 @@ out:
  	return ret;
  }
 
+
+static int erofs_xattr_filter_hashbit(struct xattr_item *item)
+{
+	u8 prefix = item->prefix;
+	const char *key = item->kvbuf;
+	unsigned int len = item->len[0];
+	char *name = NULL;
+	uint32_t hashbit;
+
+	if (prefix & EROFS_XATTR_LONG_PREFIX) {
+		struct ea_type_node *tnode;
+		u16 prefix_len;
+		int ret;
+
+		list_for_each_entry(tnode, &ea_name_prefixes, list) {
+			if (tnode->index == item->prefix) {
+				ret = asprintf(&name, "%s%.*s",
+					       tnode->type.prefix, len, key);
+				if (ret < 0)
+					return -ENOMEM;
+				break;
+			}
+		}
+		if (!name)
+			return -ENOENT;
+
+		if (!match_base_prefix(name, &prefix, &prefix_len)) {
+			free(name);
+			return -ENOENT;
+		}
+		key = name + prefix_len;
+		len = strlen(key);
+	}
+
+	hashbit = xxh32(key, len, EROFS_XATTR_FILTER_SEED + prefix) &
+		  (EROFS_XATTR_FILTER_BITS - 1);
+	if (name)
+		free(name);
+	return hashbit;
+}
+

lib/xattr.c out: · line 837

Quoted 10 lines
lib/xattr.c +10 -0
+static u32 erofs_xattr_filter_map(struct list_head *ixattrs)
+{
+	struct inode_xattr_node *node, *n;
+	u32 name_filter;
+	int hashbit;
+
+	name_filter = 0;
+	list_for_each_entry_safe(node, n, ixattrs, list) {
+		hashbit = erofs_xattr_filter_hashbit(node->item);
+		if (hashbit < 0)

I'd suggest to clear feature bit instead:

erofs_sb_clear_xattr_filter(&sbi);

Quoted 10 lines
lib/xattr.c +6 -0
+			return 0;
+		name_filter |= (1UL << hashbit);
+	}
+	return EROFS_XATTR_FILTER_DEFAULT & ~name_filter;
+}
+
  char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size)
  {
  	struct inode_xattr_node *node, *n;

lib/xattr.c char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size) · line 859

Quoted 5 lines
lib/xattr.c +2 -0
@@ -797,6 +855,11 @@ char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size)
  	header = (struct erofs_xattr_ibody_header *)buf;
  	header->h_shared_count = 0;
 
+	if (cfg.c_xattr_name_filter) {
+		u32 name_filter = erofs_xattr_filter_map(ixattrs);

Leave a blank line here.

lib/xattr.c char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size) · line 860

Quoted 1 line
lib/xattr.c +1 -0
+		header->h_name_filter = cpu_to_le32(name_filter);

Or

		header->h_name_filter =
			cpu_to_le32(erofs_xattr_filter_map(ixattrs));

Thanks,
Gao Xiang

author
#3 lore raw

Re: [PATCH v6 1/3] erofs-utils: add xxh32 library

In reply to Gao Xiang
Quoted 26 lines · Gao Xiang
Quoted 18 lines · Jingbo Xu

Add xxh32 library which could be used by following xattr bloom filter feature.

---
  include/erofs/xxhash.h |  72 ++++++++++++++++++++++++ lib/Makefile.am        |   3 +-
  lib/xxhash.c           | 125 +++++++++++++++++++++++++++++++++++++++++

  3 files changed, 199 insertions(+), 1 deletion(-)

  create mode 100644 include/erofs/xxhash.h
  create mode 100644 lib/xxhash.c

include/erofs/xxhash.h new file +0 -0
new file mode 100644
index 0000000..5459bd8

include/erofs/xxhash.h line 1

Quoted 1 line
include/erofs/xxhash.h new file +1 -0
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only

You should use:
/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */

Quoted 2 lines

for each header, but I guess we could drop the copyright below since it's just a declaration?

Okay.

Quoted 57 lines
Quoted 57 lines
include/erofs/xxhash.h new file +3 -0
+/*
+ * The xxhash is copied from the linux kernel at:
+ *   

https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/xxhash.c
+ *
+ * The original copyright is:
+ *
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License
(http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the
+ *     distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */

include/erofs/xxhash.h new file +4 -0
+
+#ifndef __EROFS_XXHASH_H
+#define __EROFS_XXHASH_H
+
Quoted 8 lines

include/erofs/xxhash.h line 55

Quoted 6 lines
include/erofs/xxhash.h new file +6 -0
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "defs.h"

#include <stdint.h> ?

Alright.

Quoted 94 lines
Quoted 94 lines
include/erofs/xxhash.h new file +17 -0
+
+/**
+ * xxh32() - calculate the 32-bit hash of the input with a given seed.
+ *
+ * @input:  The data to hash.
+ * @length: The length of the data to hash.
+ * @seed:   The seed can be used to alter the result predictably.
+ *
+ * Return:  The 32-bit hash of the data.
+ */
+uint32_t xxh32(const void *input, size_t length, uint32_t seed);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
lib/Makefile.am +1 -0
index 7a5dc03..3e09516 100644
@@ -24,6 +24,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
         $(top_srcdir)/include/erofs/xattr.h \
         $(top_srcdir)/include/erofs/compress_hints.h \
         $(top_srcdir)/include/erofs/fragments.h \
+      $(top_srcdir)/include/erofs/xxhash.h \

        $(top_srcdir)/lib/liberofs_private.h
    noinst_HEADERS += compressor.h

lib/Makefile.am +0 -0
@@ -31,7 +32,7 @@ liberofs_la_SOURCES = config.c io.c cache.c super.c
 inode.c xattr.c exclude.c \

                namei.c data.c compress.c compressor.c zmap.c decompress.c \
                compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
                fragments.c rb_tree.c dedupe.c uuid_unparse.c uuid.c tar.c \
-              block_list.c
+              block_list.c xxhash.c
    liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include if ENABLE_LZ4

lib/xxhash.c new file +4 -0
new file mode 100644
index 0000000..3d77fe3
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only
+/*
+ * The xxhash is copied from the linux kernel at:
+ *   

https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/lib/xxhash.c
+ *
+ * The original copyright is:
+ *
+ * xxHash - Extremely Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet.
+ *
+ * BSD 2-Clause License
(http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above
+ *     copyright notice, this list of conditions and the following disclaimer
+ *     in the documentation and/or other materials provided with the
+ *     distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License version 2 as published by the
+ * Free Software Foundation. This program is dual-licensed; you may select
+ * either version 2 of the GNU General Public License ("GPL") or BSD license
+ * ("BSD").

Quoted 8 lines
Quoted 6 lines

+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://cyan4973.github.io/xxHash/
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */

#include "erofs/defs.h"

Alright.

Quoted 6 lines

lib/xxhash.c

Quoted 3 lines
lib/xxhash.c new file +2 -0
+#include "erofs/xxhash.h"
+

Thanks,
Gao Xiang

Signature · 2 lines
Thanks,
Jingbo
author
#6 lore raw

Re: [PATCH v6 2/3] erofs-utils: update on-disk format for xattr name filter

In reply to Gao Xiang
Quoted 15 lines · Gao Xiang
Quoted 3 lines · Jingbo Xu

The xattr name bloom filter feature is going to be introduced to speed up the negative xattr lookup, e.g. system.posix_acl_[access|default] lookup when running "ls -lR" workload.

I think you could just say
"Let's keep in sync with kernel commit <12-char commit id>."

here.

...

Quoted 8 lines
Quoted 1 line

---

...

Quoted 1 line

+#define EROFS_XATTR_NAME_LEN_MAX    UCHAR_MAX

where is EROFS_XATTR_NAME_LEN_MAX used?

Indeed it's not used and I can't remember where it's used in the initial implementation...

I will drop it later.

Signature · 2 lines
Thanks,
Jingbo
author
#9 lore raw

Re: [PATCH v6 3/3] erofs-utils: mkfs: enable xattr name filter

In reply to Gao Xiang
Quoted 105 lines · Gao Xiang
Quoted 89 lines · Jingbo Xu

Introduce "-Exattr-name-filter" option to enable the xattr name bloom filter feature.

---
  include/erofs/config.h   |  1 +
  include/erofs/internal.h |  1 +
  lib/xattr.c              | 63 ++++++++++++++++++++++++++++++++++++++++ mkfs/main.c              |  7 +++++

  4 files changed, 72 insertions(+)
include/erofs/config.h +1 -0
index 8f52d2c..c51f0cd 100644
@@ -53,6 +53,7 @@ struct erofs_configure {
       bool c_ignore_mtime;
       bool c_showprogress;
       bool c_extra_ea_name_prefixes;
+    bool c_xattr_name_filter;
     #ifdef HAVE_LIBSELINUX
       struct selabel_handle *sehnd;
include/erofs/internal.h +1 -0
index 3e73eef..382024a 100644
@@ -139,6 +139,7 @@ EROFS_FEATURE_FUNCS(fragments, incompat,
 INCOMPAT_FRAGMENTS)
   EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
   EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
   EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
+EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)
     #define EROFS_I_EA_INITED    (1 << 0)
   #define EROFS_I_Z_INITED    (1 << 1)
lib/xattr.c +1 -0
index 46a301a..325241d 100644
@@ -18,6 +18,7 @@
   #include "erofs/cache.h"
   #include "erofs/io.h"
   #include "erofs/fragments.h"
+#include "erofs/xxhash.h"
   #include "liberofs_private.h"
     #define EA_HASHTABLE_BITS 16
@@ -783,6 +784,63 @@ out:
       return ret;
   }

  +

lib/xattr.c +40 -0
+static int erofs_xattr_filter_hashbit(struct xattr_item *item)
+{
+    u8 prefix = item->prefix;
+    const char *key = item->kvbuf;
+    unsigned int len = item->len[0];
+    char *name = NULL;
+    uint32_t hashbit;
+
+    if (prefix & EROFS_XATTR_LONG_PREFIX) {
+        struct ea_type_node *tnode;
+        u16 prefix_len;
+        int ret;
+
+        list_for_each_entry(tnode, &ea_name_prefixes, list) {
+            if (tnode->index == item->prefix) {
+                ret = asprintf(&name, "%s%.*s",
+                           tnode->type.prefix, len, key);
+                if (ret < 0)
+                    return -ENOMEM;
+                break;
+            }
+        }
+        if (!name)
+            return -ENOENT;
+
+        if (!match_base_prefix(name, &prefix, &prefix_len)) {
+            free(name);
+            return -ENOENT;
+        }
+        key = name + prefix_len;
+        len = strlen(key);
+    }
+
+    hashbit = xxh32(key, len, EROFS_XATTR_FILTER_SEED + prefix) &
+          (EROFS_XATTR_FILTER_BITS - 1);
+    if (name)
+        free(name);
+    return hashbit;
+}
+

lib/xattr.c out: · line 837

Quoted 10 lines
lib/xattr.c +10 -0
+static u32 erofs_xattr_filter_map(struct list_head *ixattrs)
+{
+    struct inode_xattr_node *node, *n;
+    u32 name_filter;
+    int hashbit;
+
+    name_filter = 0;
+    list_for_each_entry_safe(node, n, ixattrs, list) {
+        hashbit = erofs_xattr_filter_hashbit(node->item);
+        if (hashbit < 0)

I'd suggest to clear feature bit instead:

Quoted 1 line

erofs_sb_clear_xattr_filter(&sbi);

Good idea.

Quoted 20 lines
Quoted 12 lines

+            return 0;
+        name_filter |= (1UL << hashbit);
+    }
+    return EROFS_XATTR_FILTER_DEFAULT & ~name_filter;

lib/xattr.c +2 -0
+}
+

  char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size)
  {
      struct inode_xattr_node *node, *n;

lib/xattr.c char *erofs_export_xattr_ibody(struct list_head · line 859

Quoted 5 lines
lib/xattr.c +1 -0
@@ -797,6 +855,11 @@ char *erofs_export_xattr_ibody(struct list_head
 *ixattrs, unsigned int size)
       header = (struct erofs_xattr_ibody_header *)buf;
       header->h_shared_count = 0;
   +    if (cfg.c_xattr_name_filter) {
+        u32 name_filter = erofs_xattr_filter_map(ixattrs);

Leave a blank line here.

Quoted 6 lines

lib/xattr.c char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size) · line 860

Quoted 1 line
lib/xattr.c +1 -0
+        header->h_name_filter = cpu_to_le32(name_filter);

Or
        header->h_name_filter = cpu_to_le32(erofs_xattr_filter_map(ixattrs));

Okay. I would prefer the latter.

Signature · 2 lines
Thanks,
Jingbo