erofs-utils: code for superblock checksum calculation.
PATCH 1/1 erofs-utils: code for superblock checksum calculation.
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.
2)Add superblock checksum calculation(-o sb_cksum) as suboption. 3)Calculate superblock checksum if feature is enabled.
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 18 deletions(-)
include/erofs/config.h +1 -0
@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags; };extern struct erofs_configure cfg;
include/erofs_fs.h +23 -17
@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS EROFS_REQUIREMENT_LZ4_0PADDING +/* +* feature definations. +*/ +#define EROFS_FEATURE_SB_CHKSUM 0x0001 + +#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */ -/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files - f_favail) */ - -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ -/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */ +/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files - f_favail) */ +/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ]; } __packed;/* 128 bytes */
mkfs/main.c +52 -1
101 lines not on this page · raw message
Signature · 1 line
2.9.3
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.
Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.
ditto. and I think we can enable sbcrc by default since it is a compat feature, and add "-O nosbcrc" to disable it.
Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>
And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 11 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +0
-0
include/erofs/config.h struct erofs_configure { · line 25
Quoted 4 lines
include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;
we can add this to sbi like requirements...
Quoted 44 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +14
-17@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS EROFS_REQUIREMENT_LZ4_0PADDING +/* +* feature definations. +*/ +#define EROFS_FEATURE_SB_CHKSUM 0x0001 + +#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */ -/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files - f_favail) */ - -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ -/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */ +/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files - f_favail) */
include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];
Why modifying the above?
mkfs/main.c line 16
Quoted 11 lines
include/erofs_fs.h +0
-0} __packed;/* 128 bytes */ mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h>
I have no idea that we should introduce "zlib" just for crc32c currently... Maybe we can add some independent crc32 function..
Thanks,
Gao Xiang
Quoted 106 lines
mkfs/main.c +51
-2#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) {int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])return 0 ; } +u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { +erofs_err ("Invalid offset of checksum field: %d" , offset); +return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh, .meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh, sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3
Hi Gao,
Yes I will make the suboption naming similar to that of mk2fs.
The reason I changed the position of 'checksum' field :
Since we are calculating the checksum of erofs_super_block structure and storing it in the same structure; we cannot include
this field for actual crc calculations. Keeping it at the end makes it easy for me to calculate length of the data of which
checksum needs to be calculated. I saw similar logic in other filesystems like ext4.
We can write our own crc32() function. :) There is no problem. I thought zlib already provides one & we can use it.
anyways , I will write.
--Pratik.
Quoted 243 lines · Gao Xiang
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.ditto. and I think we can enable sbcrc by default since it is a compat feature,
and add "-O nosbcrc" to disable it.Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 4 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53+++++++++++++++++++++++++++++++++++++++++++++++++-
include/erofs/config.h struct erofs_configure { · line 25
Quoted 11 lines
3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;we can add this to sbi like requirements...
include/erofs_fs.h line 21
Quoted 10 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +0
-0@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS
EROFS_REQUIREMENT_LZ4_0PADDINGQuoted 4 lines
include/erofs_fs.h +3
-0+/* +* feature definations. +*/ include/erofs_fs.h
Quoted 10 lines
include/erofs_fs.h +5
-3+#define EROFS_FEATURE_SB_CHKSUM 0x0001 + +#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +0
-4-/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h
Quoted 12 lines
include/erofs_fs.h +2
-10- -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ -/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +4
-0+/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];Why modifying the above?
mkfs/main.c line 16
Quoted 11 lines
include/erofs_fs.h +0
-0} __packed;/* 128 bytes */ mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h> I have no idea that we should introduce "zlib" just for crc32c currently... Maybe we can add some independent crc32 function..
Thanks,
Gao Xiangmkfs/main.c
Quoted 9 lines
mkfs/main.c +1
-0#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct
erofs_super_block))mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 31 lines
mkfs/main.c +22
-0@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 9 lines
mkfs/main.c +1
-1{int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 13 lines
mkfs/main.c +6
-0return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 22 lines
mkfs/main.c +15
-0return 0 ; } +u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { +erofs_err ("Invalid offset of checksum field: %d" , offset); +return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 8 lines
mkfs/main.c +1
-0.meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,Quoted 14 lines
mkfs/main.c +5
-1sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3
Hi Pratik,
Quoted 2 lines · Pratik Shinde
Hi Gao,
Yes I will make the suboption naming similar to that of mk2fs.
Great! :)
Quoted 9 lines
The reason I changed the position of 'checksum' field :
Since we are calculating the checksum of
erofs_super_blockstructure and storing it in the same structure; we cannot include
this field for actual crc calculations. Keeping it at the end makes it easy for me to calculate length of the data of which
checksum needs to be calculated. I saw similar logic in other filesystems like ext4.
No, that is just they didn't add checksum field at the beginning of its design. I think you can leave chksum to 0, and calculate chksum and fill it, to be specfic:
In the erofs-utils,
1) fill .checksum to 0;
2) calculate crc32c of the entire erofs_super_block;
3) fill the real checksum to .checksum;
In the kernel,
1) read .checksum to a variable;
2) fill .checksum to 0;
3) calculate crc32c of the entire erofs_super_block;
4) compare the given one and the calculated one;
That is all :)
Quoted 3 lines
We can write our own
crc32()function. :) There is no problem. I thought zlib already provides one & we can use it.
I think we can use crc32c() since I already wrote comments for erofs in 4.19.
Looking forward to your next version :)
Thanks,
Gao Xiang
Quoted 249 lines
anyways , I will write.
--Pratik.
Quoted 243 lines · Gao Xiang
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.ditto. and I think we can enable sbcrc by default since it is a compat feature,
and add "-O nosbcrc" to disable it.Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 4 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53+++++++++++++++++++++++++++++++++++++++++++++++++-
include/erofs/config.h struct erofs_configure { · line 25
Quoted 11 lines
3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;we can add this to sbi like requirements...
include/erofs_fs.h line 21
Quoted 10 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +0
-0@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS
EROFS_REQUIREMENT_LZ4_0PADDINGQuoted 4 lines
include/erofs_fs.h +3
-0+/* +* feature definations. +*/ include/erofs_fs.h
Quoted 10 lines
include/erofs_fs.h +5
-3+#define EROFS_FEATURE_SB_CHKSUM 0x0001 + +#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +0
-4-/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h
Quoted 12 lines
include/erofs_fs.h +2
-10- -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ -/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +4
-0+/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];Why modifying the above?
mkfs/main.c line 16
Quoted 11 lines
include/erofs_fs.h +0
-0} __packed;/* 128 bytes */ mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h> I have no idea that we should introduce "zlib" just for crc32c currently... Maybe we can add some independent crc32 function..
Thanks,
Gao Xiangmkfs/main.c
Quoted 9 lines
mkfs/main.c +1
-0#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct
erofs_super_block))mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 31 lines
mkfs/main.c +22
-0@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 9 lines
mkfs/main.c +1
-1{int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 13 lines
mkfs/main.c +6
-0return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 22 lines
mkfs/main.c +15
-0return 0 ; } +u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { +erofs_err ("Invalid offset of checksum field: %d" , offset); +return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 8 lines
mkfs/main.c +1
-0.meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,Quoted 14 lines
mkfs/main.c +5
-1sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3
Quoted 26 lines · Gao Xiang via Linux-erofs
Hi Pratik,
Quoted 2 lines · Pratik Shinde
Hi Gao,
Yes I will make the suboption naming similar to that of mk2fs.Great! :)
Quoted 9 lines
The reason I changed the position of 'checksum' field :
Since we are calculating the checksum of
erofs_super_blockstructure and storing it in the same structure; we cannot include
this field for actual crc calculations. Keeping it at the end makes it easy for me to calculate length of the data of which
checksum needs to be calculated. I saw similar logic in other filesystems like ext4.No, that is just they didn't add checksum field at the beginning of its design. I think you can leave chksum to 0, and calculate chksum and fill it, to be specfic:
In the erofs-utils,
1) fill .checksum to 0;
2) calculate crc32c of the entireerofs_super_block;
3) fill the real checksum to .checksum;
Quoted 5 lines
In the kernel,
1) read .checksum to a variable;
2) fill .checksum to 0;
3) calculate crc32c of the entireerofs_super_block;
4) compare the given one and the calculated one;
That's one way, FYI, the way used in ext4/f2fs now is:
- calc [0, checksum]'s chksum
- use above chksum as seed of chksum within range [chksum + chksum_size, end]
- fill chksum value in range [checksum, chksum + chksum_size]
Thanks,
Quoted 263 lines
That is all :)
Quoted 3 lines
We can write our own
crc32()function. :) There is no problem. I thought zlib already provides one & we can use it.I think we can use
crc32c()since I already wrote comments for erofs in 4.19.Looking forward to your next version :)
Thanks,
Gao XiangQuoted 249 lines
anyways , I will write.
--Pratik.
Quoted 243 lines · Gao Xiang
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.ditto. and I think we can enable sbcrc by default since it is a compat feature,
and add "-O nosbcrc" to disable it.Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 4 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53+++++++++++++++++++++++++++++++++++++++++++++++++-
include/erofs/config.h struct erofs_configure { · line 25
Quoted 11 lines
3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;we can add this to sbi like requirements...
include/erofs_fs.h line 21
Quoted 10 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +0
-0@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS
EROFS_REQUIREMENT_LZ4_0PADDINGQuoted 4 lines
include/erofs_fs.h +3
-0+/* +* feature definations. +*/ include/erofs_fs.h
Quoted 10 lines
include/erofs_fs.h +5
-3+#define EROFS_FEATURE_SB_CHKSUM 0x0001 + +#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +0
-4-/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h
Quoted 12 lines
include/erofs_fs.h +2
-10- -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ -/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only */
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +4
-0+/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];Why modifying the above?
mkfs/main.c line 16
Quoted 11 lines
include/erofs_fs.h +0
-0} __packed;/* 128 bytes */ mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h> I have no idea that we should introduce "zlib" just for crc32c currently... Maybe we can add some independent crc32 function..
Thanks,
Gao Xiangmkfs/main.c
Quoted 9 lines
mkfs/main.c +1
-0#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct
erofs_super_block))mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 31 lines
mkfs/main.c +22
-0@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 9 lines
mkfs/main.c +1
-1{int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 13 lines
mkfs/main.c +6
-0return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 22 lines
mkfs/main.c +15
-0return 0 ; } +u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { +erofs_err ("Invalid offset of checksum field: %d" , offset); +return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 8 lines
mkfs/main.c +1
-0.meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,Quoted 14 lines
mkfs/main.c +5
-1sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3
Thanks Chao,
Initially I was of doing the same, but then I thought changing the position of cheksum field to either start OR end
will be much simpler.
I think I will go with Gao's method. thereby not changing the layout of super-block.
So just to be clear, I will be writing our own crc32c() function correct ?
--Pratik.
Quoted 314 lines · Chao Yu
Quoted 14 lines · Gao Xiang via Linux-erofs
Hi Pratik,
Quoted 2 lines · Pratik Shinde
Hi Gao,
Yes I will make the suboption naming similar to that of mk2fs.Great! :)
Quoted 6 lines
The reason I changed the position of 'checksum' field :
Since we are calculating the checksum of
erofs_super_blockstructure and storing it in the same structure; we cannot include
this field for actual crc calculations. Keeping it at the end makes iteasy
Quoted 2 lines
Quoted 2 lines
for me to calculate length of the data of which checksum needs to be calculated. I saw similar logic in other
filesystems
Quoted 3 lines
Quoted 1 line
like ext4.
No, that is just they didn't add checksum field at the beginning of its
design.
Quoted 1 line
I think you can leave chksum to 0, and calculate chksum and fill it, to
be specfic:
Quoted 11 lines
In the erofs-utils,
1) fill .checksum to 0;
2) calculate crc32c of the entireerofs_super_block;
3) fill the real checksum to .checksum;In the kernel,
1) read .checksum to a variable;
2) fill .checksum to 0;
3) calculate crc32c of the entireerofs_super_block;
4) compare the given one and the calculated one;That's one way, FYI, the way used in ext4/f2fs now is:
- calc [0, checksum]'s chksum
- use above chksum as seed of chksum within range [chksum +chksum_size, end]
- fill chksum value in range [checksum, chksum +chksum_size]Thanks,
Quoted 8 lines
That is all :)
Quoted 3 lines
We can write our own
crc32()function. :) There is no problem. I thought zlib already provides one & we can use it.I think we can use
crc32c()since I already wrote comments for erofs in4.19.
Quoted 74 lines
Looking forward to your next version :)
Thanks,
Gao XiangQuoted 68 lines
anyways , I will write.
--Pratik.
Quoted 62 lines · Gao Xiang
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.ditto. and I think we can enable sbcrc by default since it is a compat feature,
and add "-O nosbcrc" to disable it.Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 4 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53+++++++++++++++++++++++++++++++++++++++++++++++++-
include/erofs/config.h struct erofs_configure { · line 25
Quoted 11 lines
3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;we can add this to sbi like requirements...
include/erofs_fs.h line 21
Quoted 10 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +0
-0@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS
EROFS_REQUIREMENT_LZ4_0PADDINGQuoted 1 line
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
include/erofs_fs.h +5
-0+/* +* feature definations. +*/ +#define EROFS_FEATURE_SB_CHKSUM 0x0001 +Quoted 8 lines
Quoted 8 lines
include/erofs_fs.h
Quoted 8 lines
include/erofs_fs.h +3
-3+#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*/
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +0
-4-/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
include/erofs_fs.h +0
-4- -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ Quoted 8 lines
Quoted 8 lines
include/erofs_fs.h
Quoted 8 lines
include/erofs_fs.h +2
-6-/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only
Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
*/
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +4
-0+/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];Why modifying the above?
Quoted 2 lines
}
__packed; /* 128 bytes */Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
mkfs/main.c line 16
Quoted 9 lines
mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h> I have no idea that we should introduce "zlib" just for crc32c
currently...
Quoted 74 lines
Quoted 74 lines
Quoted 74 lines
Maybe we can add some independent crc32 function..
Thanks,
Gao Xiangmkfs/main.c
Quoted 9 lines
mkfs/main.c +1
-0#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct
erofs_super_block))mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 31 lines
mkfs/main.c +22
-0@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 9 lines
mkfs/main.c +1
-1{int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 13 lines
mkfs/main.c +6
-0return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
Quoted 3 lines
mkfs/main.c +0
-0return 0 ; }Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[])
Quoted 8 lines
mkfs/main.c +8
-0+u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { + erofs_err("Invalid offset of checksum field: %d",offset);
Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 11 lines
mkfs/main.c +7
-0+return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 8 lines
mkfs/main.c +1
-0.meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,Quoted 14 lines
mkfs/main.c +5
-1sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3
Hi Pratik, and Chao,
Quoted 5 lines · Pratik Shinde
Thanks Chao,
Initially I was of doing the same, but then I thought changing the position of cheksum field to either start OR end
will be much simpler.
To Pratik: erofs layout is almost determined from linux-4.19, and it is fine
for this chksum field;
Quoted 3 lines
I think I will go with Gao's method. thereby not changing the layout of super-block.
So just to be clear, I will be writing our owncrc32c()function correct ?
Yes, just go with crc32c(). and it's very simple to implement :)
Quoted 47 lines
--Pratik.
Quoted 42 lines · Chao Yu
Quoted 14 lines · Gao Xiang via Linux-erofs
Hi Pratik,
Quoted 2 lines · Pratik Shinde
Hi Gao,
Yes I will make the suboption naming similar to that of mk2fs.Great! :)
Quoted 6 lines
The reason I changed the position of 'checksum' field :
Since we are calculating the checksum of
erofs_super_blockstructure and storing it in the same structure; we cannot include
this field for actual crc calculations. Keeping it at the end makes iteasy
Quoted 2 lines
Quoted 2 lines
for me to calculate length of the data of which checksum needs to be calculated. I saw similar logic in other
filesystems
Quoted 3 lines
Quoted 1 line
like ext4.
No, that is just they didn't add checksum field at the beginning of its
design.
Quoted 1 line
I think you can leave chksum to 0, and calculate chksum and fill it, to
be specfic:
Quoted 11 lines
In the erofs-utils,
1) fill .checksum to 0;
2) calculate crc32c of the entireerofs_super_block;
3) fill the real checksum to .checksum;In the kernel,
1) read .checksum to a variable;
2) fill .checksum to 0;
3) calculate crc32c of the entireerofs_super_block;
4) compare the given one and the calculated one;That's one way, FYI, the way used in ext4/f2fs now is:
- calc [0, checksum]'s chksum
- use above chksum as seed of chksum within range [chksum +chksum_size, end]
- fill chksum value in range [checksum, chksum +chksum_size]
To Chao: Yes, I think that's ok as well, but we can do much simple in EROFS,
and it won't break crc32c pipeline... (calculate in one crc32c function) :)
Thanks,
Gao Xiang
Quoted 272 lines
Quoted 272 lines
Thanks,
Quoted 8 lines
That is all :)
Quoted 3 lines
We can write our own
crc32()function. :) There is no problem. I thought zlib already provides one & we can use it.I think we can use
crc32c()since I already wrote comments for erofs in4.19.
Quoted 74 lines
Looking forward to your next version :)
Thanks,
Gao XiangQuoted 68 lines
anyways , I will write.
--Pratik.
Quoted 62 lines · Gao Xiang
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Adding code for superblock checksum calculation.
This patch adds following things:
1)Handle suboptions('-o') to mkfs utility.Thanks for your patch. :)
Can we use "-O feature" instead in order to keep in line with mke2fs?
Quoted 1 line
2)Add superblock checksum
calculation(-o sb_cksum)as suboption.ditto. and I think we can enable sbcrc by default since it is a compat feature,
and add "-O nosbcrc" to disable it.Quoted 3 lines
3)Calculate superblock checksum if feature is enabled.
Signed-off-by Pratik Shinde <[email protected]>And could you please also read my following comments and fix them. and I'd like to accept your erofs-utils modification in advance. :)
But now you can see we are moving EROFS out of staging now as the "real" part of Linux, this is the fundamental stuff of other new features if we want to develop more actively... So we can wait for the final result and add this new feature to kernel then...
Quoted 4 lines
--- include/erofs/config.h | 1 + include/erofs_fs.h | 40 +++++++++++++++++++++---------------- mkfs/main.c | 53+++++++++++++++++++++++++++++++++++++++++++++++++-
include/erofs/config.h struct erofs_configure { · line 25
Quoted 11 lines
3 files changed, 76 insertions(+), 18 deletions(-)include/erofs/config.h +1
-0@@ -22,6 +22,7 @@ struct erofs_configure {char *c_src_path;char *c_compr_alg_master;int c_compr_level_master; +int c_feature_flags;we can add this to sbi like requirements...
include/erofs_fs.h line 21
Quoted 10 lines
include/erofs/config.h +0
-0};extern struct erofs_configure cfg;include/erofs_fs.h +0
-0@@ -20,25 +20,31 @@#define EROFS_REQUIREMENT_LZ4_0PADDING 0x00000001#define EROFS_ALL_REQUIREMENTS
EROFS_REQUIREMENT_LZ4_0PADDINGQuoted 1 line
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
include/erofs_fs.h +5
-0+/* +* feature definations. +*/ +#define EROFS_FEATURE_SB_CHKSUM 0x0001 +Quoted 8 lines
Quoted 8 lines
include/erofs_fs.h
Quoted 8 lines
include/erofs_fs.h +3
-3+#define EROFS_HAS_COMPAT_FEATURE (super,mask) \ + ( le32_to_cpu((super)->features) & (mask) ) +struct erofs_super_block {/* 0 */ __le32 magic;/* in the little endian */ -/* 4 */ __le32 checksum;/* crc32c(super_block) */ -/* 8 */ __le32 features;/* (aka. feature_compat) */ -/* 12 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
*/
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +0
-4-/* 13 */ __u8 reserved; - -/* 14 */ __le16 root_nid; -/* 16 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */Quoted 12 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
include/erofs_fs.h +0
-4- -/* 24 */ __le64 build_time;/* inode v1 time derivation */ -/* 32 */ __le32 build_time_nsec; -/* 36 */ __le32 blocks;/* used for statfs */ Quoted 8 lines
Quoted 8 lines
include/erofs_fs.h
Quoted 8 lines
include/erofs_fs.h +2
-6-/* 40 */ __le32 meta_blkaddr; -/* 44 */ __le32 xattr_blkaddr; -/* 48 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ -/* 64 */ __u8 volume_name[16 ];/* volume name */ -/* 80 */ __le32 requirements;/* (aka. feature_incompat) */ - +/* 4 */ __le32 features;/* (aka. feature_compat) */ +/* 8 */ __u8 blkszbits;/* support block_size == PAGE_SIZE only
Quoted 21 lines
Quoted 21 lines
Quoted 21 lines
*/
include/erofs_fs.h
Quoted 4 lines
include/erofs_fs.h +4
-0+/* 9 */ __u8 reserved; + +/* 10 */ __le16 root_nid; +/* 12 */ __le64 inos;/* total valid ino # (== f_files -
f_favail) */include/erofs_fs.h line 48
Quoted 10 lines
include/erofs_fs.h +9
-0+/* 20 */ __le64 build_time;/* inode v1 time derivation */ +/* 28 */ __le32 build_time_nsec; +/* 32 */ __le32 blocks;/* used for statfs */ +/* 36 */ __le32 meta_blkaddr; +/* 40 */ __le32 xattr_blkaddr; +/* 44 */ __u8 uuid[16 ];/* 128-bit uuid for volume */ +/* 60 */ __u8 volume_name[16 ];/* volume name */ +/* 76 */ __le32 requirements;/* (aka. feature_incompat) */ +/* 80 */ __le32 checksum;/* crc32c(super_block) */ /* 84 */ __u8 reserved2[44 ];Why modifying the above?
Quoted 2 lines
}
__packed; /* 128 bytes */Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
mkfs/main.c line 16
Quoted 9 lines
mkfs/main.c +1
-0@@ -13,12 +13,14 @@#include <limits.h> #include <libgen.h> #include <sys/stat.h> +#include <zlib.h> I have no idea that we should introduce "zlib" just for crc32c
currently...
Quoted 74 lines
Quoted 74 lines
Quoted 74 lines
Maybe we can add some independent crc32 function..
Thanks,
Gao Xiangmkfs/main.c
Quoted 9 lines
mkfs/main.c +1
-0#include "erofs/config.h" #include "erofs/print.h" #include "erofs/cache.h" #include "erofs/inode.h" #include "erofs/io.h" #include "erofs/compress.h" +#include "erofs/defs.h" #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct
erofs_super_block))mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 31 lines
mkfs/main.c +22
-0@@ -31,6 +33,28 @@ static void usage(void)fprintf (stderr," -EX[,...] X=extended options\n" ); } +char *feature_opts[] = { +"sb_cksum" ,NULL +}; +#define O_SB_CKSUM 0 + +static int parse_feature_subopts (char *opts) +{ +char *arg; + +while (*opts !='\0' ) { +switch (getsubopt (&opts, feature_opts, &arg)) { +case O_SB_CKSUM : + cfg.c_feature_flags |=EROFS_FEATURE_SB_CHKSUM ; +break ; +default : +erofs_err ("incorrect suboption" ); +return -EINVAL ; + } + } +return 0 ; +} +static int parse_extended_opts (const char *opts) {#define MATCH_EXTENTED_OPT (opt, token, keylen) \@@ -79,7 +103,7 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 9 lines
mkfs/main.c +1
-1{int opt, i; -while ((opt =getopt (argc, argv,"d:z:E:" )) !=-1 ) { +while ((opt =getopt (argc, argv,"d:z:E:o:" )) !=-1 ) {switch (opt) {case 'z' :if (!optarg) {@@ -113,6 +137,12 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char
Quoted 13 lines
mkfs/main.c +6
-0return opt;break ; +case 'o' : + opt =parse_feature_subopts (optarg); +if (opt) +return opt; +break ; +default :/* '?' */ return -EINVAL ; }@@ -144,6 +174,21 @@ static int mkfs_parse_options_cfg(int argc, char*argv[])
Quoted 3 lines
mkfs/main.c +0
-0return 0 ; }Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
mkfs/main.c static int mkfs_parse_options_cfg(int argc, char *argv[])
Quoted 8 lines
mkfs/main.c +8
-0+u32 erofs_superblock_checksum (struct erofs_super_block *sb) +{ +int offset; +u32 crc; + + offset = offsetof(struct erofs_super_block , checksum); +if (offset <0 || offset >sizeof (struct erofs_super_block )) { + erofs_err("Invalid offset of checksum field: %d",offset);
Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 11 lines
mkfs/main.c +7
-0+return -1 ; + } + crc =crc32 (~0 , (const unsigned char *)sb,(size_t )offset); +erofs_dump ("superblock checksum: 0x%x\n" , crc); +return 0 ; +} +int erofs_mkfs_update_super_block (struct erofs_buffer_head *bh,erofs_nid_t root_nid) {@@ -155,6 +200,7 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,mkfs/main.c int erofs_mkfs_update_super_block(struct
Quoted 8 lines
mkfs/main.c +1
-0.meta_blkaddr = sbi.meta_blkaddr, .xattr_blkaddr = 0, .requirements = cpu_to_le32(sbi.requirements), + .features = cpu_to_le32(cfg.c_feature_flags), };const unsigned int sb_blksize =round_up (EROFS_SUPER_END ,EROFS_BLKSIZ );@@ -169,6 +215,11 @@ int erofs_mkfs_update_super_block(struct
erofs_buffer_head*bh,Quoted 14 lines
mkfs/main.c +5
-1sb.blocks =cpu_to_le32 (erofs_mapbh (NULL , true)); sb.root_nid =cpu_to_le16 (root_nid); +if (EROFS_HAS_COMPAT_FEATURE (&sb,EROFS_FEATURE_SB_CHKSUM )) { +u32 crc =erofs_superblock_checksum (&sb); + sb.checksum =cpu_to_le32 (crc); + } + buf =calloc (sb_blksize,1 );if (!buf) { erofs_err("Failed to allocate memory for sb: %s", --2.9.3