Skip to contentSkip to conversation

erofs-utils: code for superblock checksum calculation.

PATCH In review Pratik Shinde 3 participants 12 messages Aug 24, 2019 – Oct 9, 2019

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 erofs-utils: code for superblock checksum calculation.

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64 +++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
 	char *c_src_path;
 	char *c_compr_alg_master;
 	int c_compr_level_master;
+	int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +10 -0
index 601b477..9ac2635 100644
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING	0x00000001
 #define EROFS_ALL_REQUIREMENTS		EROFS_REQUIREMENT_LZ4_0PADDING
 
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES		EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM		0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask)	\
+	( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +63 -1

100 lines not on this page · raw message

Signature · 1 line
2.9.3
#1 lore raw

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 152 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64 +++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
 	char *c_src_path;
 	char *c_compr_alg_master;
 	int c_compr_level_master;
+	int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +10 -0
index 601b477..9ac2635 100644
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING	0x00000001
 #define EROFS_ALL_REQUIREMENTS		EROFS_REQUIREMENT_LZ4_0PADDING
 
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES		EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM		0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask)	\
+	( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +63 -1
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
 	fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
+#define CRCPOLY	0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+	int i;
+	u32 crc = seed;
+
+	while (len--) {
+		crc ^= *in++;
+		for (i = 0; i < 8; i++) {
+			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+		}
+	}
+	erofs_dump("calculated crc: 0x%x\n", crc);
+	return crc;
+}
+
+char *feature_opts[] = {
+	"nosbcrc", NULL
+};
+#define O_SB_CKSUM	0
+
+static int parse_feature_subopts(char *opts)
+{
+	char *arg;
+
+	cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+	while (*opts != '\0') {
+		switch(getsubopt(&opts, feature_opts, &arg)) {
+		case O_SB_CKSUM:
+			cfg.c_feature_flags |= (~EROFS_FEATURE_SB_CHKSUM);
+			break;
+		default:
+			erofs_err("incorrect suboption");
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 {
 	int opt, i;
 
-	while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+	cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+	while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
 		switch (opt) {
 		case 'z':
 			if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 				return opt;
 			break;
 
+		case 'O':
+			opt = parse_feature_subopts(optarg);
+			if (opt)
+				return opt;
+			break;
+
 		default: /* '?' */
 			return -EINVAL;
 		}
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+	u32 crc;
+	crc = crc32c(~0, (const unsigned char *)sb,
+		    sizeof(struct erofs_super_block));
+	erofs_dump("superblock checksum: 0x%x\n", crc);
+	return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 				  erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 		.meta_blkaddr  = sbi.meta_blkaddr,
 		.xattr_blkaddr = 0,
 		.requirements = cpu_to_le32(sbi.requirements),
+		.features = cpu_to_le32(cfg.c_feature_flags),
 	};
 	const unsigned int sb_blksize =
 		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 	sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
 	sb.root_nid     = cpu_to_le16(root_nid);
 
+	if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+		sb.checksum = 0;
+		u32 crc = erofs_superblock_checksum(&sb);
+		sb.checksum = cpu_to_le32(crc);
+	}
+
 	buf = calloc(sb_blksize, 1);
 	if (!buf) {
 		erofs_err("Failed to allocate memory for sb: %s",
Signature · 1 line
2.9.3
#2 lore raw
Quoted 29 lines · Gao Xiang via Linux-erofs

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

Quoted 8 lines

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and

^ it should be a stop sign (.)

Sorry... that's my keyboard fault...

Let's postpone all new features and address more about moving out EROFS. Only in this way, EROFS could have good future or it would be dead. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 157 lines

I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 152 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64 +++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
 	char *c_src_path;
 	char *c_compr_alg_master;
 	int c_compr_level_master;
+	int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +10 -0
index 601b477..9ac2635 100644
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING	0x00000001
 #define EROFS_ALL_REQUIREMENTS		EROFS_REQUIREMENT_LZ4_0PADDING
 
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES		EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM		0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask)	\
+	( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +63 -1
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
 	fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
+#define CRCPOLY	0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+	int i;
+	u32 crc = seed;
+
+	while (len--) {
+		crc ^= *in++;
+		for (i = 0; i < 8; i++) {
+			crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+		}
+	}
+	erofs_dump("calculated crc: 0x%x\n", crc);
+	return crc;
+}
+
+char *feature_opts[] = {
+	"nosbcrc", NULL
+};
+#define O_SB_CKSUM	0
+
+static int parse_feature_subopts(char *opts)
+{
+	char *arg;
+
+	cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+	while (*opts != '\0') {
+		switch(getsubopt(&opts, feature_opts, &arg)) {
+		case O_SB_CKSUM:
+			cfg.c_feature_flags |= (~EROFS_FEATURE_SB_CHKSUM);
+			break;
+		default:
+			erofs_err("incorrect suboption");
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 {
 	int opt, i;
 
-	while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+	cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+	while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
 		switch (opt) {
 		case 'z':
 			if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 				return opt;
 			break;
 
+		case 'O':
+			opt = parse_feature_subopts(optarg);
+			if (opt)
+				return opt;
+			break;
+
 		default: /* '?' */
 			return -EINVAL;
 		}
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 	return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+	u32 crc;
+	crc = crc32c(~0, (const unsigned char *)sb,
+		    sizeof(struct erofs_super_block));
+	erofs_dump("superblock checksum: 0x%x\n", crc);
+	return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 				  erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 		.meta_blkaddr  = sbi.meta_blkaddr,
 		.xattr_blkaddr = 0,
 		.requirements = cpu_to_le32(sbi.requirements),
+		.features = cpu_to_le32(cfg.c_feature_flags),
 	};
 	const unsigned int sb_blksize =
 		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
 	sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
 	sb.root_nid     = cpu_to_le16(root_nid);
 
+	if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+		sb.checksum = 0;
+		u32 crc = erofs_superblock_checksum(&sb);
+		sb.checksum = cpu_to_le32(crc);
+	}
+
 	buf = calloc(sb_blksize, 1);
 	if (!buf) {
 		erofs_err("Failed to allocate memory for sb: %s",
Signature · 1 line
2.9.3
#3 lore raw
In reply to Gao Xiang via Linux-erofs

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

One more thing, can we still send non feature patches?

--Pratik

Quoted 202 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 65 lines
include/erofs_fs.h +10 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES               EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +39 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |= (~EROFS_FEATURE_SB_CHKSUM);
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +6 -0
                              return opt;
                      break;
 
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 15 lines
mkfs/main.c +6 -1
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#4 lore raw

Hi Pratik,

Quoted 3 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now.

Thank you.

Quoted 3 lines

Once erofs makes it 'fs/' please do reconsider implementing it.

One more thing, can we still send non feature patches?

Yes, of course. All bugfix patches for erofs kernel and erofs-utils are OK, if it's important, I will apply it for 5.4 round immediately.

Thanks,
Gao Xiang

Quoted 208 lines

--Pratik

Quoted 202 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 65 lines
include/erofs_fs.h +10 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES               EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +39 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |= (~EROFS_FEATURE_SB_CHKSUM);
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +6 -0
                              return opt;
                      break;
 
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 15 lines
mkfs/main.c +6 -1
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#5 lore raw
In reply to Pratik Shinde

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at mount time, and for large images, this feature can be used to checksum the super block only (and then use verficiation subsystem to verify on-the-fly in the future...)

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 210 lines

One more thing, can we still send non feature patches?

--Pratik

Quoted 202 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 65 lines
include/erofs_fs.h +10 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES               EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +39 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t len)
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |= (~EROFS_FEATURE_SB_CHKSUM);
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +6 -0
                              return opt;
                      break;
 
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 15 lines
mkfs/main.c +6 -1
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#6 lore raw

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your approach is good. Let me go through it one more time and reply back.

--Pratik

Quoted 243 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at mount time,
and for large images, this feature can be used to checksum the super block only
(and then use verficiation subsystem to verify on-the-fly in the future...)

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 99 lines

One more thing, can we still send non feature patches?

--Pratik

Quoted 91 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 22 lines
include/erofs_fs.h +10 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES               EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t

len)

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
mkfs/main.c +20 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void)

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 81 lines
Quoted 81 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +9 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +6 -0
                              return opt;
                      break;
 
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 15 lines
mkfs/main.c +6 -1
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#7 lore raw

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your approach is good. Let me go through it one more time and reply back.

Thanks for your reply and interest :)
I think we can complete all pending features together if you have some time on this stuff. (fsdebug utility is on the way as well...)

BTW, I'm now investigating new high CR algorithm (very likely XZ) as well, it will be likely a RFC version in this round for wider scenarios and later decompression subsystem.

Preliminary TODO lists will be discussed in this year China Linux Storage & Filesystem workshop (next week) and will be posted to mailing lists for further wider discussion (if more folks have interest in developing it) as well. :)

Thanks,
Gao Xiang

Quoted 248 lines

--Pratik

Quoted 243 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at mount time,
and for large images, this feature can be used to checksum the super block only
(and then use verficiation subsystem to verify on-the-fly in the future...)

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 99 lines

One more thing, can we still send non feature patches?

--Pratik

Quoted 91 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 22 lines
include/erofs_fs.h +10 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES               EROFS_FEATURE_SB_CHKSUM
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in, size_t

len)

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
mkfs/main.c +20 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY : 0);
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void)

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 81 lines
Quoted 81 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +9 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc, char

Quoted 13 lines
mkfs/main.c +6 -0
                              return opt;
                      break;
 
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc, char

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 15 lines
mkfs/main.c +6 -1
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb, EROFS_FEATURE_SB_CHKSUM)) {
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#8 lore raw

Hello Gao,

Yes I would like to work on pending features. Also please let us know the new development items.

--Pratik

Quoted 283 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your approach is good. Let me go through it one more time and reply back.

Thanks for your reply and interest :)
I think we can complete all pending features together if you have some time on this stuff. (fsdebug utility is on the way as well...)

BTW, I'm now investigating new high CR algorithm (very likely XZ) as well, it will be likely a RFC version in this round for wider scenarios and later decompression subsystem.

Preliminary TODO lists will be discussed in this year China Linux Storage & Filesystem workshop (next week) and will be posted to mailing lists for further wider discussion (if more folks have interest in developing it) as well. :)

Thanks,
Gao Xiang

Quoted 20 lines

--Pratik

Quoted 15 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at mount time,
and for large images, this feature can be used to checksum the super

block

Quoted 2 lines
Quoted 2 lines

only
(and then use verficiation subsystem to verify on-the-fly in the

future...)

Quoted 91 lines
Quoted 91 lines

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 78 lines

One more thing, can we still send non feature patches?

--Pratik

Quoted 70 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 1 line
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

include/erofs_fs.h

Quoted 4 lines
include/erofs_fs.h +4 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES

EROFS_FEATURE_SB_CHKSUM

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
include/erofs_fs.h +6 -0
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in,

size_t

Quoted 9 lines
Quoted 9 lines

len)

Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void)

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY :

0);

Quoted 21 lines
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
mkfs/main.c +12 -0
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void)

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
mkfs/main.c +5 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 8 lines
mkfs/main.c +4 -0
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
                              return opt;
                      break;
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 10 lines
mkfs/main.c +6 -0
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,

Quoted 4 lines
mkfs/main.c +1 -0
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb,

EROFS_FEATURE_SB_CHKSUM)) {

Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
mkfs/main.c +5 -1
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#9 lore raw

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hello Gao,

Yes I would like to work on pending features. Also please let us know the new development items.

Thanks, I will post later after gathering the first round suggestions to mailing lists. We have time to improve this filesystem. :)

Thanks,
Gao Xiang

Quoted 288 lines

--Pratik

Quoted 283 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your approach is good. Let me go through it one more time and reply back.

Thanks for your reply and interest :)
I think we can complete all pending features together if you have some time on this stuff. (fsdebug utility is on the way as well...)

BTW, I'm now investigating new high CR algorithm (very likely XZ) as well, it will be likely a RFC version in this round for wider scenarios and later decompression subsystem.

Preliminary TODO lists will be discussed in this year China Linux Storage & Filesystem workshop (next week) and will be posted to mailing lists for further wider discussion (if more folks have interest in developing it) as well. :)

Thanks,
Gao Xiang

Quoted 20 lines

--Pratik

Quoted 15 lines · Gao Xiang,

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for now. Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at mount time,
and for large images, this feature can be used to checksum the super

block

Quoted 2 lines
Quoted 2 lines

only
(and then use verficiation subsystem to verify on-the-fly in the

future...)

Quoted 91 lines
Quoted 91 lines

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 78 lines

One more thing, can we still send non feature patches?

--Pratik

Quoted 70 lines · Gao Xiang,

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 1 line
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

include/erofs_fs.h

Quoted 4 lines
include/erofs_fs.h +4 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES

EROFS_FEATURE_SB_CHKSUM

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
include/erofs_fs.h +6 -0
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in,

size_t

Quoted 9 lines
Quoted 9 lines

len)

Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 43

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY :

0);

Quoted 21 lines
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
mkfs/main.c +12 -0
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 63

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
mkfs/main.c +5 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 8 lines
mkfs/main.c +4 -0
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

*argv[])

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 10 lines
mkfs/main.c +2 -1
 {
      int opt, i;
 
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
                              return opt;
                      break;
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int argc,

Quoted 10 lines
mkfs/main.c +6 -0
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int argc,

char

Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 27 lines

*argv[])

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 16 lines
mkfs/main.c +9 -0
      return 0;
 }
 
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct · line 228

Quoted 4 lines
mkfs/main.c +1 -0
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb,

EROFS_FEATURE_SB_CHKSUM)) {

Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
mkfs/main.c +5 -1
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb: %s",
--

2.9.3

#10 lore raw

Hello Gao,

I have few questions regarding the checksum feature:
1) Are we still keeping it as a feature. (i.e enabled by default . but can be disabled during mkfs)
2) For small images we are going to cksum the entire image. what could be the maximum size of 'small' images.

also, I think we don't have routines to read buffers back into the memory. I will write them first and send a patch.

-Pratik

Quoted 315 lines · Gao Xiang

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hello Gao,

Yes I would like to work on pending features. Also please let us know the new development items.

Thanks, I will post later after gathering the first round suggestions to mailing lists. We have time to improve this filesystem. :)

Thanks,
Gao Xiang

Quoted 11 lines

--Pratik

Quoted 6 lines · Gao Xiang,

Hi Pratik,

Quoted 3 lines · Pratik Shinde

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your

approach is

Quoted 22 lines
Quoted 22 lines
Quoted 1 line

good. Let me go through it one more time and reply back.

Thanks for your reply and interest :)
I think we can complete all pending features together if you have some time on this stuff. (fsdebug utility is on the way as well...)

BTW, I'm now investigating new high CR algorithm (very likely XZ) as well, it will be likely a RFC version in this round for wider scenarios and later decompression subsystem.

Preliminary TODO lists will be discussed in this year China Linux Storage & Filesystem workshop (next week) and will be posted to mailing lists for further wider discussion (if more folks have interest in developing it) as well. :)

Thanks,
Gao Xiang

Quoted 3 lines

--Pratik

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines · Gao Xiang,
Quoted 7 lines

Hi Pratik,

Quoted 3 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for

now.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line

Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at

mount

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

time,
and for large images, this feature can be used to checksum the

super

Quoted 24 lines
Quoted 24 lines

block

Quoted 2 lines
Quoted 2 lines

only
(and then use verficiation subsystem to verify on-the-fly in the

future...)

Quoted 20 lines
Quoted 20 lines

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 7 lines

One more thing, can we still send non feature patches?

--Pratik

On Sat, 24 Aug, 2019, 7:30 PM Gao Xiang, <[email protected]>

Quoted 97 lines · wrote:
Quoted 97 lines
Quoted 71 lines
Quoted 71 lines
Quoted 71 lines
Quoted 71 lines

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 1 line
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

include/erofs_fs.h

Quoted 4 lines
include/erofs_fs.h +4 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES

EROFS_FEATURE_SB_CHKSUM

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
include/erofs_fs.h +6 -0
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in,

size_t

Quoted 1 line
Quoted 1 line

len)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 43

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ?

CRCPOLY :

Quoted 22 lines
Quoted 22 lines

0);

Quoted 21 lines
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
mkfs/main.c +12 -0
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 63

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
mkfs/main.c +5 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 8 lines
mkfs/main.c +4 -0
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
 {
      int opt, i;
 
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 7 lines
mkfs/main.c +2 -1
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
                              return opt;
                      break;
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 10 lines
mkfs/main.c +6 -0
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
      return 0;
 }
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int · line 202

Quoted 10 lines
mkfs/main.c +9 -0
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head

*bh,

Quoted 18 lines
Quoted 18 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 3 lines
mkfs/main.c +0 -0
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct · line 228

Quoted 4 lines
mkfs/main.c +1 -0
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb,

EROFS_FEATURE_SB_CHKSUM)) {

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct · line 236

Quoted 8 lines
mkfs/main.c +5 -0
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb:

%s",

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines

--
2.9.3

#11 lore raw

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Hello Gao,

I have few questions regarding the checksum feature:
1) Are we still keeping it as a feature. (i.e enabled by default . but can be disabled during mkfs)

Yes, I think it should be a new compatible feature suggested by Richard and enabled by default (this feature should be marked in on-disk super_block...)

Quoted 2 lines

2) For small images we are going to cksum the entire image. what could be the maximum size of 'small' images.

I think we can checksum the first 4k for now in mkfs by default (or only checksum 4k as the first step), and then it can be configured to a larger number n*4k. (But for the kernel side, it has to be fully implemented due to forward compatibility.)

Quoted 3 lines

also, I think we don't have routines to read buffers back into the memory. I will write them first and send a patch.

Yes, I remembered that you once wrote a similar function, that is ok. And it's no rush, we have more than a month to develop pending features for linux-5.5. Just a reminder because we can prepare them for now :)

Thanks,
Gao Xiang

Quoted 320 lines

-Pratik

Quoted 315 lines · Gao Xiang

Hi Pratik,

Quoted 4 lines · Pratik Shinde

Hello Gao,

Yes I would like to work on pending features. Also please let us know the new development items.

Thanks, I will post later after gathering the first round suggestions to mailing lists. We have time to improve this filesystem. :)

Thanks,
Gao Xiang

Quoted 11 lines

--Pratik

Quoted 6 lines · Gao Xiang,

Hi Pratik,

Quoted 3 lines · Pratik Shinde

Hi Gao,

Yes I can work on it. Sorry I missed this mail. I think your

approach is

Quoted 22 lines
Quoted 22 lines
Quoted 1 line

good. Let me go through it one more time and reply back.

Thanks for your reply and interest :)
I think we can complete all pending features together if you have some time on this stuff. (fsdebug utility is on the way as well...)

BTW, I'm now investigating new high CR algorithm (very likely XZ) as well, it will be likely a RFC version in this round for wider scenarios and later decompression subsystem.

Preliminary TODO lists will be discussed in this year China Linux Storage & Filesystem workshop (next week) and will be posted to mailing lists for further wider discussion (if more folks have interest in developing it) as well. :)

Thanks,
Gao Xiang

Quoted 3 lines

--Pratik

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines · Gao Xiang,
Quoted 7 lines

Hi Pratik,

Quoted 3 lines · Pratik Shinde

Hi Gao,

I completely understand your concern.You can drop this patch for

now.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line

Once erofs makes it 'fs/' please do reconsider implementing it.

I think we can work on this pending feature for v5.5 now. My idea is to add an extra field in erofs_super_block to indicate the number of blocks (4k) for checking.

So for small images, this feature can checksum the whole image at

mount

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

time,
and for large images, this feature can be used to checksum the

super

Quoted 24 lines
Quoted 24 lines

block

Quoted 2 lines
Quoted 2 lines

only
(and then use verficiation subsystem to verify on-the-fly in the

future...)

Quoted 20 lines
Quoted 20 lines

The following workflow is for erofs-utils, I think it's
1) erofs_mkfs_update_super_block with checksum = 0
2) erofs_bflush(NULL)
3) reread the corresponding blocks and calculate checksum;
4) write checksum to erofs_super_block;

Does it sound reasonable? and do you still have interest and time for this? Looking forword to your reply...

Thanks,
Gao Xiang

Quoted 7 lines

One more thing, can we still send non feature patches?

--Pratik

On Sat, 24 Aug, 2019, 7:30 PM Gao Xiang, <[email protected]>

Quoted 97 lines · wrote:
Quoted 97 lines
Quoted 71 lines
Quoted 71 lines
Quoted 71 lines
Quoted 71 lines

Hi Pratik,

Quoted 5 lines · Pratik Shinde

Adding code for superblock checksum calculation.

incorporated the changes suggested in previous patch.

Thanks for your v2 patch.

Actually, I have some concern about the length of checksum, sizeof(struct erofs_super_block) could be changed in the later version, it's bad for EROFS future scalablity.

And I tend not to add another on-disk field to record the size of erofs_super_block as well, because the old Linux kernel cannot handle more about the new size, so it has little use except for checksum calculation.

Few hours ago, I discussed with Chao about this concern, I think this feature can be changed to do multiple-block checksum at the mount time, e.g:
- for small images, we can check the whole image once at the mount time;
- for the large image, we can check the superblock at the mount time, the rest can be handled by block-based verification layer.

But we agreed that don't add this for this round
since it's quite a new feature.

All in all, it's a new feature since we are addressing moving out of staging for this round. I tend to postpone this feature for now. I understand that you are very interested in EROFS. Considering EROFS current staging status, it's not such a place to add new features at all! I have marked your patch down and I will work with you later. Hope to get your understanding...

Thanks,
Gao Xiang

Quoted 4 lines
---
 include/erofs/config.h |  1 +
 include/erofs_fs.h     | 10 ++++++++
 mkfs/main.c            | 64

+++++++++++++++++++++++++++++++++++++++++++++++++-

Quoted 19 lines
 3 files changed, 74 insertions(+), 1 deletion(-)
include/erofs/config.h +1 -0
index 05fe6b2..40cd466 100644
@@ -22,6 +22,7 @@ struct erofs_configure {
      char *c_src_path;
      char *c_compr_alg_master;
      int c_compr_level_master;
+     int c_feature_flags;
 };
 
 extern struct erofs_configure cfg;
include/erofs_fs.h +0 -0
index 601b477..9ac2635 100644

include/erofs_fs.h line 21

Quoted 2 lines
include/erofs_fs.h +0 -0
@@ -20,6 +20,16 @@
 #define EROFS_REQUIREMENT_LZ4_0PADDING       0x00000001
 #define EROFS_ALL_REQUIREMENTS

EROFS_REQUIREMENT_LZ4_0PADDING

Quoted 1 line
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

include/erofs_fs.h

Quoted 4 lines
include/erofs_fs.h +4 -0
+/*
+ * feature definations.
+ */
+#define EROFS_DEFAULT_FEATURES

EROFS_FEATURE_SB_CHKSUM

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
include/erofs_fs.h +6 -0
+#define EROFS_FEATURE_SB_CHKSUM              0x0001
+
+
+#define EROFS_HAS_COMPAT_FEATURE(super,mask) \
+     ( le32_to_cpu((super)->features) & (mask) )
+
 struct erofs_super_block {
 /*  0 */__le32 magic;           /* in the little endian */
 /*  4 */__le32 checksum;        /* crc32c(super_block) */
mkfs/main.c +0 -0
index f127fe1..355fd2c 100644
@@ -31,6 +31,45 @@ static void usage(void)
      fprintf(stderr, " -EX[,...] X=extended options\n");
 }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

mkfs/main.c static void usage(void) · line 35

Quoted 2 lines
mkfs/main.c +2 -0
+#define CRCPOLY      0x82F63B78
+static inline u32 crc32c(u32 seed, unsigned char const *in,

size_t

Quoted 1 line
Quoted 1 line

len)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 43

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     int i;
+     u32 crc = seed;
+
+     while (len--) {
+             crc ^= *in++;
+             for (i = 0; i < 8; i++) {
+                     crc = (crc >> 1) ^ ((crc & 1) ?

CRCPOLY :

Quoted 22 lines
Quoted 22 lines

0);

Quoted 21 lines
Quoted 21 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
mkfs/main.c +12 -0
+             }
+     }
+     erofs_dump("calculated crc: 0x%x\n", crc);
+     return crc;
+}
+
+char *feature_opts[] = {
+     "nosbcrc", NULL
+};
+#define O_SB_CKSUM   0
+
+static int parse_feature_subopts(char *opts)
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static void usage(void) · line 63

Quoted 8 lines
mkfs/main.c +8 -0
+{
+     char *arg;
+
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while (*opts != '\0') {
+             switch(getsubopt(&opts, feature_opts, &arg)) {
+             case O_SB_CKSUM:
+                     cfg.c_feature_flags |=

(~EROFS_FEATURE_SB_CHKSUM);

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
mkfs/main.c +5 -0
+                     break;
+             default:
+                     erofs_err("incorrect suboption");
+                     return -EINVAL;
+             }
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 8 lines
mkfs/main.c +4 -0
+     }
+     return 0;
+}
+
 static int parse_extended_opts(const char *opts)
 {
 #define MATCH_EXTENTED_OPT(opt, token, keylen) \
@@ -79,7 +118,8 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
 {
      int opt, i;
 
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 7 lines
mkfs/main.c +2 -1
-     while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+     cfg.c_feature_flags = EROFS_DEFAULT_FEATURES;
+     while ((opt = getopt(argc, argv, "d:z:E:o:")) != -1) {
              switch (opt) {
              case 'z':
                      if (!optarg) {
@@ -113,6 +153,12 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
                              return opt;
                      break;
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int

Quoted 10 lines
mkfs/main.c +6 -0
+             case 'O':
+                     opt = parse_feature_subopts(optarg);
+                     if (opt)
+                             return opt;
+                     break;
+
              default: /* '?' */
                      return -EINVAL;
              }
@@ -144,6 +190,15 @@ static int mkfs_parse_options_cfg(int

argc,

Quoted 5 lines
Quoted 5 lines

char

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

*argv[])

Quoted 3 lines
mkfs/main.c +0 -0
      return 0;
 }
 
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines

mkfs/main.c static int mkfs_parse_options_cfg(int · line 202

Quoted 10 lines
mkfs/main.c +9 -0
+u32 erofs_superblock_checksum(struct erofs_super_block *sb)
+{
+     u32 crc;
+     crc = crc32c(~0, (const unsigned char *)sb,
+                 sizeof(struct erofs_super_block));
+     erofs_dump("superblock checksum: 0x%x\n", crc);
+     return crc;
+}
+
 int erofs_mkfs_update_super_block(struct erofs_buffer_head

*bh,

Quoted 18 lines
Quoted 18 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 3 lines
mkfs/main.c +0 -0
                                erofs_nid_t root_nid)
 {
@@ -155,6 +210,7 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

mkfs/main.c int erofs_mkfs_update_super_block(struct

Quoted 8 lines
mkfs/main.c +1 -0
              .meta_blkaddr  = sbi.meta_blkaddr,
              .xattr_blkaddr = 0,
              .requirements = cpu_to_le32(sbi.requirements),
+             .features = cpu_to_le32(cfg.c_feature_flags),
      };
      const unsigned int sb_blksize =
              round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -169,6 +225,12 @@ int erofs_mkfs_update_super_block(struct

erofs_buffer_head *bh,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct · line 228

Quoted 4 lines
mkfs/main.c +1 -0
      sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
      sb.root_nid     = cpu_to_le16(root_nid);
 
+     if (EROFS_HAS_COMPAT_FEATURE(&sb,

EROFS_FEATURE_SB_CHKSUM)) {

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

mkfs/main.c int erofs_mkfs_update_super_block(struct · line 236

Quoted 8 lines
mkfs/main.c +5 -0
+             sb.checksum = 0;
+             u32 crc = erofs_superblock_checksum(&sb);
+             sb.checksum = cpu_to_le32(crc);
+     }
+
      buf = calloc(sb_blksize, 1);
      if (!buf) {
              erofs_err("Failed to allocate memory for sb:

%s",

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines

--
2.9.3