Skip to contentSkip to conversation

erofs-utils: introduce exclude dirs and files

PATCH In review Li Guifu 3 participants 4 messages Jan 23, 2020 – Feb 3, 2020 Library code

lore
Share & tools
lore

Li Guifu

author
#0 lore raw

PATCH 1/1 erofs-utils: introduce exclude dirs and files

From: Li Guifu <[email protected]>

Add support exlcude directory when build image with long option arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg -e dir1/dir2,dir11/dir22 that seperated by ','

---
 include/erofs/exclude.h |  28 ++++++++
 lib/Makefile.am         |   2 +-
 lib/exclude.c           | 149 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  10 +++
 mkfs/main.c             |  13 +++-
 5 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c
include/erofs/exclude.h new file +28 -0
new file mode 100644
index 0000000..3f9fb4d
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h
+ * Created by Li Guifu <[email protected]>
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {
+	struct list_head list;
+	struct list_head hlist;
+	char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+	return list_empty(&e->hlist);
+}
+#endif
+
lib/Makefile.am +1 -1
index 1ff81f9..e4b51e6 100644
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/exclude.c new file +149 -0

149 lines not on this page · raw message

lib/inode.c +10 -0
index bd0652b..6278ff8 100644
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -825,6 +826,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 	DIR *_dir;
 	struct dirent *dp;
 	struct erofs_dentry *d;
+	struct rule_entry *e;
 
 	ret = erofs_prepare_xattr_ibody(dir->i_srcpath, &dir->i_xattrs);
 	if (ret < 0)
@@ -863,6 +865,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		return ERR_PTR(-errno);
 	}
 
+	e = erofs_pattern_matched(dir->i_srcpath + strlen(cfg.c_src_path));
 	while (1) {
 		/*
 		 * set errno to 0 before calling readdir() in order to
@@ -876,7 +879,14 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		if (is_dot_dotdot(dp->d_name) ||
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
+		if (e) {
+			struct rule_entry *le;
 
+			le = erofs_entry_matched(e, dp->d_name,
+						 strlen(dp->d_name));
+			if (le && erofs_is_pattern_end(le))
+				continue;
+		}
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
mkfs/main.c +12 -1
index 817a6c1..ab718cb 100644
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 'e'},
 	{0, 0, 0, 0},
 };
 
@@ -127,7 +129,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	char *endptr;
 	int opt, i;
 
-	while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+	while((opt = getopt_long(argc, argv, "d:x:z:E:T:e:",
 				 long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'z':
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 'e':
+			if (erofs_parse_exclude_path(optarg)) {
+				usage();
+				exit(0);
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -334,6 +343,7 @@ int main(int argc, char **argv)
 	struct timeval t;
 
 	erofs_init_configure();
+	erofs_init_rules();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
 
 	cfg.c_legacy_compress = false;
@@ -429,6 +439,7 @@ exit:
 	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
+	erofs_free_rules();
 
 	if (err) {
 		erofs_err("\tCould not format the device : %s\n",
Signature · 1 line
2.17.1
#1 lore raw

Hi Guifu,

Quoted 2 lines · Li Guifu

From: Li Guifu <[email protected]>

I could imagine that gmail is hard to work on mainland China, but what's wrong with @foxmail.com this time? It could be better to use some email address from a stable provider.

Quoted 4 lines

Add support exlcude directory when build image with long option arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg -e dir1/dir2,dir11/dir22 that seperated by ','

How about the following commit message?

Add excluded file feature "--exclude-path=", which can be used to build EROFS image without some user specific files or dirs. Such multiple files can be seperated by ','.

and I tend to avoid the short option '-e' in order to keep it for later use (e.g. file system encryption).

Quoted 18 lines
---
 include/erofs/exclude.h |  28 ++++++++
 lib/Makefile.am         |   2 +-
 lib/exclude.c           | 149 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  10 +++
 mkfs/main.c             |  13 +++-
 5 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c
include/erofs/exclude.h new file +0 -0
new file mode 100644
index 0000000..3f9fb4d

include/erofs/exclude.h line 3

Quoted 3 lines
include/erofs/exclude.h new file +3 -0
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h

erofs-utils/include/erofs/exclude.h

No backslash here

include/erofs/exclude.h line 10

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {

rule is too general here. exclude_rule_entry

include/erofs/exclude.h line 17

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+	struct list_head list;
+	struct list_head hlist;
+	char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);

Same here erofs_init_exclude_rules(), erofs_free_exclude_rules();

Quoted 31 lines
include/erofs/exclude.h new file +11 -0
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len);
 
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+	return list_empty(&e->hlist);
+}
+#endif
+
lib/Makefile.am +1 -1
index 1ff81f9..e4b51e6 100644
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/exclude.c new file +0 -0
new file mode 100644
index 0000000..25a9d32

lib/exclude.c line 3

Quoted 3 lines
lib/exclude.c new file +3 -0
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\lib\exclude.c

Same here.

lib/exclude.c line 16

Quoted 13 lines
lib/exclude.c new file +13 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct rule_entry ext_rule;

exclude_rule ?

lib/exclude.c line 18

Quoted 2 lines
lib/exclude.c new file +2 -0
+
+static struct rule_entry *new_entry(const char *s, unsigned int len)

The function name is too general as well. new_exclude_rule()

lib/exclude.c line 23

Quoted 5 lines
lib/exclude.c new file +5 -0
+{
+	struct rule_entry *e = malloc(sizeof(struct rule_entry));
+
+	if (!e)
+		return NULL;

How about using ERR_PTR(-ENOMEM) instead...

lib/exclude.c line 36

Quoted 13 lines
lib/exclude.c new file +13 -0
+
+	e->name = strndup(s, len);
+	if (!e->name)
+		goto err_strdup;
+
+	init_list_head(&e->hlist);
+	init_list_head(&e->list);
+
+	return e;
+
+err_strdup:
+	free(e);
+	return NULL;

Same here.

Quoted 12 lines
lib/exclude.c new file +12 -0
+}
+
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len)
+{
+	struct rule_entry *pos;
+
+	while (*s == '/') {
+		s++;
+		len--;
+	}
+	list_for_each_entry(pos, &e->hlist, list) {

lib/exclude.c line 58

Quoted 10 lines
lib/exclude.c new file +10 -0
+		unsigned int l = strlen(pos->name);
+
+		if (l == len && !strcmp(pos->name, s))
+			return pos;
+	}
+
+	return NULL;
+}
+
+static int add_rules(struct rule_entry *e, const char *s)

Same here.

Quoted 22 lines
lib/exclude.c new file +22 -0
+{
+	const char *ptr;
+	struct rule_entry *le;
+
+	while (*s == '/')
+		s++;
+	ptr = s;
+
+forward:
+	while(*ptr != '/' && *ptr != '\0')
+		ptr++;
+
+	le = erofs_entry_matched(e, s, ptr - s);
+	if (!le) {
+		struct rule_entry *me = new_entry(s, ptr - s);
+
+		if (!me)
+			return -ENOMEM;
+		list_add_tail(&me->list, &e->hlist);
+		le = me;
+	}
+	e = le;

lib/exclude.c line 90

Quoted 10 lines
lib/exclude.c new file +10 -0
+
+	if (*ptr++ != '\0') {
+		s = ptr;
+		goto forward;
+	}
+
+	return 0;
+}
+
+static void free_rules(struct list_head *h)

Same here.

Quoted 115 lines
lib/exclude.c new file +59 -0
+{
+	struct rule_entry *e, *n;
+
+	list_for_each_entry_safe(e, n, h, list) {
+		list_del(&e->list);
+		free_rules(&e->hlist);
+		free(e->name);
+		free(e);
+	}
+}
+void erofs_init_rules(void)
+{
+	init_list_head(&ext_rule.list);
+	init_list_head(&ext_rule.hlist);
+	ext_rule.name = "/";
+}
+
+void erofs_free_rules(void)
+{
+	free_rules(&ext_rule.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+	const char *str, *ptr = args;
+	const char sep = ',';
+
+forward:
+	while(*ptr != sep && *ptr != '\0')
+		ptr++;
+
+	str = strndup(args, ptr - args);
+	if (!str)
+		goto err_free_paths;
+
+	if (add_rules(&ext_rule, str))
+		goto err_free_paths;
+
+	if (*ptr++ != '\0') {
+		args = ptr;
+		goto forward;
+	}
+
+	return 0;
+
+err_free_paths:
+	erofs_free_rules();
+	return -ENOMEM;
+}
+
+struct rule_entry *erofs_pattern_matched(const char *s)
+{
+	unsigned int len = strlen(s);
+
+	if (!len)
+		return &ext_rule;
+
+	return erofs_entry_matched(&ext_rule, s, len);
+}
lib/inode.c +10 -0
index bd0652b..6278ff8 100644
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -825,6 +826,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 	DIR *_dir;
 	struct dirent *dp;
 	struct erofs_dentry *d;
+	struct rule_entry *e;
 
 	ret = erofs_prepare_xattr_ibody(dir->i_srcpath, &dir->i_xattrs);
 	if (ret < 0)
@@ -863,6 +865,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		return ERR_PTR(-errno);
 	}
 
+	e = erofs_pattern_matched(dir->i_srcpath + strlen(cfg.c_src_path));
 	while (1) {
 		/*
 		 * set errno to 0 before calling readdir() in order to
@@ -876,7 +879,14 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		if (is_dot_dotdot(dp->d_name) ||
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
+		if (e) {
+			struct rule_entry *le;
 
+			le = erofs_entry_matched(e, dp->d_name,
+						 strlen(dp->d_name));
+			if (le && erofs_is_pattern_end(le))
+				continue;
+		}
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
mkfs/main.c +1 -0
index 817a6c1..ab718cb 100644
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>

mkfs/main.c line 34

Quoted 4 lines
mkfs/main.c +1 -0
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 'e'},

{"exclude-path", required_argument, NULL, 2},

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 132

Quoted 9 lines
mkfs/main.c +1 -1
 	{0, 0, 0, 0},
 };
 
@@ -127,7 +129,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	char *endptr;
 	int opt, i;
 
-	while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+	while((opt = getopt_long(argc, argv, "d:x:z:E:T:e:",

Leave it unchanged.

(opt = getopt_long(argc, argv, "d:x:z:E:T:",

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 183

Quoted 8 lines
mkfs/main.c +1 -0
 				 long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'z':
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 'e':
		case 2:

Thanks,
Gao Xiang

Quoted 29 lines
mkfs/main.c +8 -0
+			if (erofs_parse_exclude_path(optarg)) {
+				usage();
+				exit(0);
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -334,6 +343,7 @@ int main(int argc, char **argv)
 	struct timeval t;
 
 	erofs_init_configure();
+	erofs_init_rules();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
 
 	cfg.c_legacy_compress = false;
@@ -429,6 +439,7 @@ exit:
 	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
+	erofs_free_rules();
 
 	if (err) {
 		erofs_err("\tCould not format the device : %s\n",
Signature · 1 line
2.17.1
author
#2 lore raw

Gao Xiang

It seems a good iead about functions' names and commit message, I will fix them asap

A value is needed to long option exclude-path in the getopt_long, Do you have a another idea ?

Quoted 374 lines · Gao Xiang

Hi Guifu,

Quoted 1 line · Li Guifu

From: Li Guifu <[email protected]>

I could imagine that gmail is hard to work on mainland China, but what's wrong with @foxmail.com this time? It could be better to use some email address from a stable provider.

Quoted 3 lines

Add support exlcude directory when build image with long option arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg -e dir1/dir2,dir11/dir22 that seperated by ','

How about the following commit message?

Add excluded file feature "--exclude-path=", which can be used to build EROFS image without some user specific files or dirs. Such multiple files can be seperated by ','.

and I tend to avoid the short option '-e' in order to keep it for later use (e.g. file system encryption).

Quoted 17 lines
---
 include/erofs/exclude.h |  28 ++++++++
 lib/Makefile.am         |   2 +-
 lib/exclude.c           | 149 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  10 +++
 mkfs/main.c             |  13 +++-
 5 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c
include/erofs/exclude.h new file +0 -0
new file mode 100644
index 0000000..3f9fb4d

include/erofs/exclude.h line 3

Quoted 3 lines
include/erofs/exclude.h new file +3 -0
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h

erofs-utils/include/erofs/exclude.h

No backslash here

include/erofs/exclude.h line 10

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {

rule is too general here. exclude_rule_entry

include/erofs/exclude.h line 17

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+	struct list_head list;
+	struct list_head hlist;
+	char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);

Same here erofs_init_exclude_rules(), erofs_free_exclude_rules();

Quoted 30 lines
include/erofs/exclude.h new file +11 -0
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+	return list_empty(&e->hlist);
+}
+#endif
+
lib/Makefile.am +1 -1
index 1ff81f9..e4b51e6 100644
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/exclude.c new file +0 -0
new file mode 100644
index 0000000..25a9d32

lib/exclude.c line 3

Quoted 3 lines
lib/exclude.c new file +3 -0
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\lib\exclude.c

Same here.

lib/exclude.c line 16

Quoted 13 lines
lib/exclude.c new file +13 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct rule_entry ext_rule;

exclude_rule ?

lib/exclude.c line 18

Quoted 2 lines
lib/exclude.c new file +2 -0
+
+static struct rule_entry *new_entry(const char *s, unsigned int len)

The function name is too general as well. new_exclude_rule()

lib/exclude.c line 23

Quoted 5 lines
lib/exclude.c new file +5 -0
+{
+	struct rule_entry *e = malloc(sizeof(struct rule_entry));
+
+	if (!e)
+		return NULL;

How about using ERR_PTR(-ENOMEM) instead...

lib/exclude.c line 36

Quoted 13 lines
lib/exclude.c new file +13 -0
+
+	e->name = strndup(s, len);
+	if (!e->name)
+		goto err_strdup;
+
+	init_list_head(&e->hlist);
+	init_list_head(&e->list);
+
+	return e;
+
+err_strdup:
+	free(e);
+	return NULL;

Same here.

Quoted 12 lines
lib/exclude.c new file +12 -0
+}
+
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len)
+{
+	struct rule_entry *pos;
+
+	while (*s == '/') {
+		s++;
+		len--;
+	}
+	list_for_each_entry(pos, &e->hlist, list) {

lib/exclude.c line 58

Quoted 10 lines
lib/exclude.c new file +10 -0
+		unsigned int l = strlen(pos->name);
+
+		if (l == len && !strcmp(pos->name, s))
+			return pos;
+	}
+
+	return NULL;
+}
+
+static int add_rules(struct rule_entry *e, const char *s)

Same here.

Quoted 22 lines
lib/exclude.c new file +22 -0
+{
+	const char *ptr;
+	struct rule_entry *le;
+
+	while (*s == '/')
+		s++;
+	ptr = s;
+
+forward:
+	while(*ptr != '/' && *ptr != '\0')
+		ptr++;
+
+	le = erofs_entry_matched(e, s, ptr - s);
+	if (!le) {
+		struct rule_entry *me = new_entry(s, ptr - s);
+
+		if (!me)
+			return -ENOMEM;
+		list_add_tail(&me->list, &e->hlist);
+		le = me;
+	}
+	e = le;

lib/exclude.c line 90

Quoted 10 lines
lib/exclude.c new file +10 -0
+
+	if (*ptr++ != '\0') {
+		s = ptr;
+		goto forward;
+	}
+
+	return 0;
+}
+
+static void free_rules(struct list_head *h)

Same here.

Quoted 115 lines
lib/exclude.c new file +59 -0
+{
+	struct rule_entry *e, *n;
+
+	list_for_each_entry_safe(e, n, h, list) {
+		list_del(&e->list);
+		free_rules(&e->hlist);
+		free(e->name);
+		free(e);
+	}
+}
+void erofs_init_rules(void)
+{
+	init_list_head(&ext_rule.list);
+	init_list_head(&ext_rule.hlist);
+	ext_rule.name = "/";
+}
+
+void erofs_free_rules(void)
+{
+	free_rules(&ext_rule.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+	const char *str, *ptr = args;
+	const char sep = ',';
+
+forward:
+	while(*ptr != sep && *ptr != '\0')
+		ptr++;
+
+	str = strndup(args, ptr - args);
+	if (!str)
+		goto err_free_paths;
+
+	if (add_rules(&ext_rule, str))
+		goto err_free_paths;
+
+	if (*ptr++ != '\0') {
+		args = ptr;
+		goto forward;
+	}
+
+	return 0;
+
+err_free_paths:
+	erofs_free_rules();
+	return -ENOMEM;
+}
+
+struct rule_entry *erofs_pattern_matched(const char *s)
+{
+	unsigned int len = strlen(s);
+
+	if (!len)
+		return &ext_rule;
+
+	return erofs_entry_matched(&ext_rule, s, len);
+}
lib/inode.c +10 -0
index bd0652b..6278ff8 100644
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -825,6 +826,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 	DIR *_dir;
 	struct dirent *dp;
 	struct erofs_dentry *d;
+	struct rule_entry *e;
 
 	ret = erofs_prepare_xattr_ibody(dir->i_srcpath, &dir->i_xattrs);
 	if (ret < 0)
@@ -863,6 +865,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		return ERR_PTR(-errno);
 	}
 
+	e = erofs_pattern_matched(dir->i_srcpath + strlen(cfg.c_src_path));
 	while (1) {
 		/*
 		 * set errno to 0 before calling readdir() in order to
@@ -876,7 +879,14 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		if (is_dot_dotdot(dp->d_name) ||
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
+		if (e) {
+			struct rule_entry *le;
 
+			le = erofs_entry_matched(e, dp->d_name,
+						 strlen(dp->d_name));
+			if (le && erofs_is_pattern_end(le))
+				continue;
+		}
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
mkfs/main.c +1 -0
index 817a6c1..ab718cb 100644
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>

mkfs/main.c line 34

Quoted 4 lines
mkfs/main.c +1 -0
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 'e'},

{"exclude-path", required_argument, NULL, 2},

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 132

Quoted 9 lines
mkfs/main.c +1 -1
 	{0, 0, 0, 0},
 };
 
@@ -127,7 +129,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	char *endptr;
 	int opt, i;
 
-	while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+	while((opt = getopt_long(argc, argv, "d:x:z:E:T:e:",

Leave it unchanged.

(opt = getopt_long(argc, argv, "d:x:z:E:T:",

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 183

Quoted 8 lines
mkfs/main.c +1 -0
 				 long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'z':
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 'e':
		case 2:

Thanks,
Gao Xiang

Quoted 29 lines
mkfs/main.c +8 -0
+			if (erofs_parse_exclude_path(optarg)) {
+				usage();
+				exit(0);
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -334,6 +343,7 @@ int main(int argc, char **argv)
 	struct timeval t;
 
 	erofs_init_configure();
+	erofs_init_rules();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
 
 	cfg.c_legacy_compress = false;
@@ -429,6 +439,7 @@ exit:
 	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
+	erofs_free_rules();
 
 	if (err) {
 		erofs_err("\tCould not format the device : %s\n",
Signature · 1 line
2.17.1
#3 lore raw
Quoted 5 lines · Li Guifu

Gao Xiang

It seems a good iead about functions' names and commit message, I will fix them asap

Yes, great!

Quoted 3 lines

A value is needed to long option exclude-path in the getopt_long, Do you have a another idea ?

I suggested a soltion in the previous email as well, see what we did for "--help" option...

quote here

Quoted 25 lines
Quoted 25 lines

mkfs/main.c line 34

Quoted 3 lines

static struct option long_options[] = {

mkfs/main.c +1 -0
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 'e'},

{"exclude-path", required_argument, NULL, 2},

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 132

Quoted 9 lines
mkfs/main.c +1 -1
 	{0, 0, 0, 0},
 };
@@ -127,7 +129,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	char *endptr;
 	int opt, i;
 
-	while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+	while((opt = getopt_long(argc, argv, "d:x:z:E:T:e:",

Leave it unchanged.

(opt = getopt_long(argc, argv, "d:x:z:E:T:",

Quoted 7 lines
mkfs/main.c +0 -0
 				 long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'z':
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 183

Quoted 1 line
mkfs/main.c +1 -0
+		case 'e':
		case 2:

Thanks,
Gao Xiang

Quoted 378 lines · Gao Xiang
Quoted 374 lines

Hi Guifu,

Quoted 1 line · Li Guifu

From: Li Guifu <[email protected]>

I could imagine that gmail is hard to work on mainland China, but what's wrong with @foxmail.com this time? It could be better to use some email address from a stable provider.

Quoted 3 lines

Add support exlcude directory when build image with long option arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg -e dir1/dir2,dir11/dir22 that seperated by ','

How about the following commit message?

Add excluded file feature "--exclude-path=", which can be used to build EROFS image without some user specific files or dirs. Such multiple files can be seperated by ','.

and I tend to avoid the short option '-e' in order to keep it for later use (e.g. file system encryption).

Quoted 17 lines
---
 include/erofs/exclude.h |  28 ++++++++
 lib/Makefile.am         |   2 +-
 lib/exclude.c           | 149 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |  10 +++
 mkfs/main.c             |  13 +++-
 5 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 include/erofs/exclude.h
 create mode 100644 lib/exclude.c
include/erofs/exclude.h new file +0 -0
new file mode 100644
index 0000000..3f9fb4d

include/erofs/exclude.h line 3

Quoted 3 lines
include/erofs/exclude.h new file +3 -0
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h

erofs-utils/include/erofs/exclude.h

No backslash here

include/erofs/exclude.h line 10

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {

rule is too general here. exclude_rule_entry

include/erofs/exclude.h line 17

Quoted 7 lines
include/erofs/exclude.h new file +7 -0
+	struct list_head list;
+	struct list_head hlist;
+	char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);

Same here erofs_init_exclude_rules(), erofs_free_exclude_rules();

Quoted 30 lines
include/erofs/exclude.h new file +11 -0
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+	return list_empty(&e->hlist);
+}
+#endif
+
lib/Makefile.am +1 -1
index 1ff81f9..e4b51e6 100644
@@ -3,7 +3,7 @@
 
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
-		      compress.c compressor.c
+		      compress.c compressor.c exclude.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
lib/exclude.c new file +0 -0
new file mode 100644
index 0000000..25a9d32

lib/exclude.c line 3

Quoted 3 lines
lib/exclude.c new file +3 -0
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\lib\exclude.c

Same here.

lib/exclude.c line 16

Quoted 13 lines
lib/exclude.c new file +13 -0
+ * Created by Li Guifu <[email protected]>
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct rule_entry ext_rule;

exclude_rule ?

lib/exclude.c line 18

Quoted 2 lines
lib/exclude.c new file +2 -0
+
+static struct rule_entry *new_entry(const char *s, unsigned int len)

The function name is too general as well. new_exclude_rule()

lib/exclude.c line 23

Quoted 5 lines
lib/exclude.c new file +5 -0
+{
+	struct rule_entry *e = malloc(sizeof(struct rule_entry));
+
+	if (!e)
+		return NULL;

How about using ERR_PTR(-ENOMEM) instead...

lib/exclude.c line 36

Quoted 13 lines
lib/exclude.c new file +13 -0
+
+	e->name = strndup(s, len);
+	if (!e->name)
+		goto err_strdup;
+
+	init_list_head(&e->hlist);
+	init_list_head(&e->list);
+
+	return e;
+
+err_strdup:
+	free(e);
+	return NULL;

Same here.

Quoted 12 lines
lib/exclude.c new file +12 -0
+}
+
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+				   const char *s, unsigned int len)
+{
+	struct rule_entry *pos;
+
+	while (*s == '/') {
+		s++;
+		len--;
+	}
+	list_for_each_entry(pos, &e->hlist, list) {

lib/exclude.c line 58

Quoted 10 lines
lib/exclude.c new file +10 -0
+		unsigned int l = strlen(pos->name);
+
+		if (l == len && !strcmp(pos->name, s))
+			return pos;
+	}
+
+	return NULL;
+}
+
+static int add_rules(struct rule_entry *e, const char *s)

Same here.

Quoted 22 lines
lib/exclude.c new file +22 -0
+{
+	const char *ptr;
+	struct rule_entry *le;
+
+	while (*s == '/')
+		s++;
+	ptr = s;
+
+forward:
+	while(*ptr != '/' && *ptr != '\0')
+		ptr++;
+
+	le = erofs_entry_matched(e, s, ptr - s);
+	if (!le) {
+		struct rule_entry *me = new_entry(s, ptr - s);
+
+		if (!me)
+			return -ENOMEM;
+		list_add_tail(&me->list, &e->hlist);
+		le = me;
+	}
+	e = le;

lib/exclude.c line 90

Quoted 10 lines
lib/exclude.c new file +10 -0
+
+	if (*ptr++ != '\0') {
+		s = ptr;
+		goto forward;
+	}
+
+	return 0;
+}
+
+static void free_rules(struct list_head *h)

Same here.

Quoted 115 lines
lib/exclude.c new file +59 -0
+{
+	struct rule_entry *e, *n;
+
+	list_for_each_entry_safe(e, n, h, list) {
+		list_del(&e->list);
+		free_rules(&e->hlist);
+		free(e->name);
+		free(e);
+	}
+}
+void erofs_init_rules(void)
+{
+	init_list_head(&ext_rule.list);
+	init_list_head(&ext_rule.hlist);
+	ext_rule.name = "/";
+}
+
+void erofs_free_rules(void)
+{
+	free_rules(&ext_rule.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+	const char *str, *ptr = args;
+	const char sep = ',';
+
+forward:
+	while(*ptr != sep && *ptr != '\0')
+		ptr++;
+
+	str = strndup(args, ptr - args);
+	if (!str)
+		goto err_free_paths;
+
+	if (add_rules(&ext_rule, str))
+		goto err_free_paths;
+
+	if (*ptr++ != '\0') {
+		args = ptr;
+		goto forward;
+	}
+
+	return 0;
+
+err_free_paths:
+	erofs_free_rules();
+	return -ENOMEM;
+}
+
+struct rule_entry *erofs_pattern_matched(const char *s)
+{
+	unsigned int len = strlen(s);
+
+	if (!len)
+		return &ext_rule;
+
+	return erofs_entry_matched(&ext_rule, s, len);
+}
lib/inode.c +10 -0
index bd0652b..6278ff8 100644
@@ -20,6 +20,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 struct erofs_sb_info sbi;
 
@@ -825,6 +826,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 	DIR *_dir;
 	struct dirent *dp;
 	struct erofs_dentry *d;
+	struct rule_entry *e;
 
 	ret = erofs_prepare_xattr_ibody(dir->i_srcpath, &dir->i_xattrs);
 	if (ret < 0)
@@ -863,6 +865,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		return ERR_PTR(-errno);
 	}
 
+	e = erofs_pattern_matched(dir->i_srcpath + strlen(cfg.c_src_path));
 	while (1) {
 		/*
 		 * set errno to 0 before calling readdir() in order to
@@ -876,7 +879,14 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
 		if (is_dot_dotdot(dp->d_name) ||
 		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
 			continue;
+		if (e) {
+			struct rule_entry *le;
 
+			le = erofs_entry_matched(e, dp->d_name,
+						 strlen(dp->d_name));
+			if (le && erofs_is_pattern_end(le))
+				continue;
+		}
 		d = erofs_d_alloc(dir, dp->d_name);
 		if (IS_ERR(d)) {
 			ret = PTR_ERR(d);
mkfs/main.c +1 -0
index 817a6c1..ab718cb 100644
@@ -21,6 +21,7 @@
 #include "erofs/io.h"
 #include "erofs/compress.h"
 #include "erofs/xattr.h"
+#include "erofs/exclude.h"
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>

mkfs/main.c line 34

Quoted 4 lines
mkfs/main.c +1 -0
@@ -30,6 +31,7 @@
 
 static struct option long_options[] = {
 	{"help", no_argument, 0, 1},
+	{"exclude-path", required_argument, NULL, 'e'},

{"exclude-path", required_argument, NULL, 2},

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 132

Quoted 9 lines
mkfs/main.c +1 -1
 	{0, 0, 0, 0},
 };
 
@@ -127,7 +129,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	char *endptr;
 	int opt, i;
 
-	while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+	while((opt = getopt_long(argc, argv, "d:x:z:E:T:e:",

Leave it unchanged.

(opt = getopt_long(argc, argv, "d:x:z:E:T:",

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[]) · line 183

Quoted 8 lines
mkfs/main.c +1 -0
 				 long_options, NULL)) != -1) {
 		switch (opt) {
 		case 'z':
@@ -178,6 +180,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			}
 			break;
 
+		case 'e':
		case 2:

Thanks,
Gao Xiang

Quoted 29 lines
mkfs/main.c +8 -0
+			if (erofs_parse_exclude_path(optarg)) {
+				usage();
+				exit(0);
+			}
+			break;
+
 		case 1:
 			usage();
 			exit(0);
@@ -334,6 +343,7 @@ int main(int argc, char **argv)
 	struct timeval t;
 
 	erofs_init_configure();
+	erofs_init_rules();
 	fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version);
 
 	cfg.c_legacy_compress = false;
@@ -429,6 +439,7 @@ exit:
 	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
+	erofs_free_rules();
 
 	if (err) {
 		erofs_err("\tCould not format the device : %s\n",
Signature · 1 line
2.17.1