erofs-utils: on-disk extent format for blocks
RFC v3 1/1 erofs-utils: on-disk extent format for blocks
1)Moved on-disk structures to erofs_fs.h
2)Some naming changes.
I think we can keep 'IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only in inode.c so it can be kept there.
what do you think ?
--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 150 insertions(+), 23 deletions(-)
include/erofs/internal.h +8 -1
@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;
@@ -93,6 +93,7 @@ struct erofs_inode {unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;
@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(int err) return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif
include/erofs_fs.h +11 -0
@@ -321,5 +321,16 @@ static inline void erofs_check_ondisk_layout_definitions(void) Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif
lib/inode.c +131 -22
198 lines not on this page · raw message
Signature · 1 line
2.9.3
Hi Pratik,
Quoted 7 lines · Pratik Shinde
1)Moved on-disk structures to
erofs_fs.h
2)Some naming changes.I think we can keep '
IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only ininode.cso it can be kept there.
what do you think ?
What I'm little concerned is the relationship between the name of IS_HOLE and its implementation...
In other words, why
roundup(start, EROFS_BLKSIZ) == start &&
roundup(end, EROFS_BLKSIZ) == end &&
(end - start) % EROFS_BLKSIZ == 0
should be an erofs hole...
But that is minor, I reserve my opinion on this for now...
This patch generally looks good to me, yet I haven't played with it till now.
Could you implement a workable RFC patch for kernel side as well? Since I'm still busying in XZ library and other convert patches for 5.6...
I'd like to hear if some other opinions from Chao and Guifu. Since it enhances the on-disk format, we need to think it over (especially its future expandability).
To Chao and Guifu,
Could you have some extra time looking at this stuff as well?
Thanks,
Gao Xiang
Quoted 273 lines
Signed-off-by Pratik Shinde <[email protected]>--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 150 insertions(+), 23 deletions(-)include/erofs/internal.h +8
-1@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;@@ -93,6 +93,7 @@ struct erofs_inode {unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(int err) return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif include/erofs_fs.h +11
-0@@ -321,5 +321,16 @@ static inline void erofs_check_ondisk_layout_definitions(void) Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif lib/inode.c +131
-22@@ -38,6 +38,97 @@ static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {struct list_head inode_hashtable[NR_INODE_HASHTABLE ]; + +#define IS_HOLE (start, end) (roundup(start, EROFS_BLKSIZ) == start && \ + roundup(end, EROFS_BLKSIZ) == end && \ + (end - start) % EROFS_BLKSIZ == 0) + +/** +read extents of the given file. +record the data extents and link them into a chain. +exclude holes present in file. +*/ +unsigned int erofs_read_sparse_extents (int fd,struct list_head *extents) +{ +erofs_blk_t startblk, endblk, datablk; +unsigned int nholes =0 ; +erofs_off_t data, hole, len =0 , last_data; +struct erofs_sparse_extent_node *e_data; + + len =lseek (fd,0 ,SEEK_END ); +if (len <0 ) +return -errno; +if (lseek (fd,0 ,SEEK_SET ) ==-1 ) +return -errno; + data =0 ; + last_data =0 ; +while (data < len) { + hole =lseek (fd, data,SEEK_HOLE ); +if (hole == len) +break ; + data =lseek (fd, hole,SEEK_DATA ); +if (data <0 || hole > data) +return -EINVAL ; +if (IS_HOLE (hole, data)) { + startblk =erofs_blknr (hole); + datablk =erofs_blknr (last_data); + endblk =erofs_blknr (data); + last_data = data; + e_data =malloc (sizeof ( +struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ; + e_data->lblk = datablk; + e_data->len = (startblk - datablk); +list_add_tail (&e_data->next, extents); + nholes += (endblk - startblk); + } + } +/* rounddown to exclude tail-end data */ +if (last_data < len && (len - last_data) >=EROFS_BLKSIZ ) { + e_data =malloc (sizeof (struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ; + startblk =erofs_blknr (last_data); + e_data->lblk = startblk; + e_data->len =erofs_blknr (rounddown ((len - last_data), +EROFS_BLKSIZ )); +list_add_tail (&e_data->next, extents); + } +return nholes; +} + +int erofs_write_sparse_extents (struct erofs_inode *inode,erofs_off_t off) +{ +struct erofs_sparse_extent_node *e_node; +struct erofs_sparse_extent_iheader *header; +char *buf; +unsigned int p =0 ; +int ret; + + buf =malloc (inode->sparse_extent_isize); +if (buf ==NULL ) +return -ENOMEM ; + header = (struct erofs_sparse_extent_iheader *) buf; + header->count =0 ; + p +=sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { +const struct erofs_sparse_extent ee = { + .ee_lblk =cpu_to_le32 (e_node->lblk), + .ee_pblk =cpu_to_le32 (e_node->pblk), + .ee_len =cpu_to_le32 (e_node->len) + }; +memcpy (buf + p, &ee,sizeof (struct erofs_sparse_extent )); + p +=sizeof (struct erofs_sparse_extent ); + header->count++; +list_del (&e_node->next); +free (e_node); + } + ret =dev_write (buf, off, inode->sparse_extent_isize); +free (buf); +return ret; +} +void erofs_inode_manager_init (void ) {unsigned int i;@@ -304,8 +395,9 @@ static bool erofs_file_is_compressible(struct erofs_inode *inode)int erofs_write_file (struct erofs_inode *inode) { -unsigned int nblocks, i; +unsigned int nblocks, i, j, nholes;int ret, fd; +struct erofs_sparse_extent_node *e_node;if (!inode->i_size) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;@@ -322,31 +414,42 @@ int erofs_write_file(struct erofs_inode *inode)/* fallback to all data uncompressed */ inode->datalayout =EROFS_INODE_FLAT_INLINE ; nblocks = inode->i_size /EROFS_BLKSIZ ; - - ret =__allocate_inode_bh_data (inode, nblocks); -if (ret) -return ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 )return -errno; - -for (i =0 ; i < nblocks; ++i) { -char buf[EROFS_BLKSIZ ]; - - ret =read (fd, buf,EROFS_BLKSIZ ); -if (ret !=EROFS_BLKSIZ ) { -if (ret <0 ) + nholes =erofs_read_sparse_extents (fd, &inode->i_sparse_extents); +if (nholes <0 ) { +close (fd); +return nholes; + } + ret =__allocate_inode_bh_data (inode, nblocks - nholes); +if (ret) { +close (fd); +return ret; + } + i = inode->u.i_blkaddr; + inode->sparse_extent_isize =sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { + inode->sparse_extent_isize +=sizeof (struct erofs_sparse_extent ); + e_node->pblk = i; + ret =lseek (fd,blknr_to_addr (e_node->lblk),SEEK_SET ); +if (ret <0 ) + goto fail; +for (j =0 ; j < e_node->len; j++) { +char buf[EROFS_BLKSIZ ]; + ret =read (fd, buf,EROFS_BLKSIZ ); +if (ret !=EROFS_BLKSIZ ) { +if (ret <0 ) + goto fail; +close (fd); +return -EAGAIN ; + } + ret =blk_write (buf, e_node->pblk + j,1 ); +if (ret) goto fail; -close (fd); -return -EAGAIN ; } - - ret =blk_write (buf, inode->u.i_blkaddr + i,1 ); -if (ret) - goto fail; + i += e_node->len; } -/* read the tail-end data */ inode->idata_size = inode->i_size %EROFS_BLKSIZ ;if (inode->idata_size) {@@ -479,8 +582,14 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)if (ret)return false;free (inode->compressmeta); + off += inode->extent_isize; } +if (inode->sparse_extent_isize) { + ret =erofs_write_sparse_extents (inode, off); +if (ret) +return false; + } inode->bh =NULL ;erofs_iput (inode);return erofs_bh_flush_generic_end (bh);@@ -737,10 +846,11 @@ struct erofs_inode *erofs_new_inode(void)init_list_head (&inode->i_subdirs);init_list_head (&inode->i_xattrs); +init_list_head (&inode->i_sparse_extents); inode->idata_size =0 ; inode->xattr_isize =0 ; - inode->extent_isize =0 ; + inode->sparse_extent_isize =0 ; inode->bh = inode->bh_inline = inode->bh_data =NULL ; inode->idata =NULL ;@@ -961,4 +1071,3 @@ struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,return erofs_mkfs_build_tree (inode); } -Signature · 1 line
2.9.3
Hi Gao,
You are correct. The macro logic can be simplified. I will do that. I will work on the kernel part of this change & do some testing on it. I will keep you posted about the change and relevant tests I am running on it.
--Pratik.
Quoted 326 lines · Gao Xiang
Hi Pratik,
Quoted 7 lines · Pratik Shinde
1)Moved on-disk structures to
erofs_fs.h
2)Some naming changes.I think we can keep '
IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only ininode.cso it can be kept there.
what do you think ?What I'm little concerned is the relationship between the name of
IS_HOLEand its implementation...In other words, why
roundup(start, EROFS_BLKSIZ)== start &&
roundup(end, EROFS_BLKSIZ)== end &&
(end - start) %EROFS_BLKSIZ== 0
should be an erofs hole...But that is minor, I reserve my opinion on this for now...
This patch generally looks good to me, yet I haven't played with it till now.
Could you implement a workable RFC patch for kernel side as well? Since I'm still busying in XZ library and other convert patches for 5.6...
I'd like to hear if some other opinions from Chao and Guifu. Since it enhances the on-disk format, we need to think it over (especially its future expandability).
To Chao and Guifu,
Could you have some extra time looking at this stuff as well?Thanks,
Gao XiangQuoted 6 lines
Signed-off-by Pratik Shinde <[email protected]>--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153++++++++++++++++++++++++++++++++++++++++-------
include/erofs_fs.h static inline void
Quoted 40 lines
3 files changed, 150 insertions(+), 23 deletions(-)include/erofs/internal.h +8
-1@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;@@ -93,6 +93,7 @@ struct erofs_inode {unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(int err) return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif include/erofs_fs.h +0
-0@@ -321,5 +321,16 @@ static inline void
erofs_check_ondisk_layout_definitions(void)lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >>
Quoted 21 lines
include/erofs_fs.h +11
-0Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif lib/inode.c +0
-0@@ -38,6 +38,97 @@ static unsigned char erofs_type_by_mode[S_IFMT >>
S_SHIFT] = {lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 5 lines
lib/inode.c +2
-0struct list_head inode_hashtable[NR_INODE_HASHTABLE ]; + +#define IS_HOLE (start, end) (roundup(start, EROFS_BLKSIZ) == start &&\lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 9 lines
lib/inode.c +9
-0+ roundup(end, EROFS_BLKSIZ) == end && \ + (end - start) % EROFS_BLKSIZ == 0) + +/** +read extents of the given file. +record the data extents and link them into a chain. +exclude holes present in file. +*/ +unsigned int erofs_read_sparse_extents(int fd, struct list_head*extents)
Quoted 40 lines
lib/inode.c +40
-0+{ +erofs_blk_t startblk, endblk, datablk; +unsigned int nholes =0 ; +erofs_off_t data, hole, len =0 , last_data; +struct erofs_sparse_extent_node *e_data; + + len =lseek (fd,0 ,SEEK_END ); +if (len <0 ) +return -errno; +if (lseek (fd,0 ,SEEK_SET ) ==-1 ) +return -errno; + data =0 ; + last_data =0 ; +while (data < len) { + hole =lseek (fd, data,SEEK_HOLE ); +if (hole == len) +break ; + data =lseek (fd, hole,SEEK_DATA ); +if (data <0 || hole > data) +return -EINVAL ; +if (IS_HOLE (hole, data)) { + startblk =erofs_blknr (hole); + datablk =erofs_blknr (last_data); + endblk =erofs_blknr (data); + last_data = data; + e_data =malloc (sizeof ( +struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ; + e_data->lblk = datablk; + e_data->len = (startblk - datablk); +list_add_tail (&e_data->next, extents); + nholes += (endblk - startblk); + } + } +/* rounddown to exclude tail-end data */ +if (last_data < len && (len - last_data) >=EROFS_BLKSIZ ) { + e_data =malloc (sizeof (struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ;lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 10 lines
lib/inode.c +10
-0+ startblk =erofs_blknr (last_data); + e_data->lblk = startblk; + e_data->len =erofs_blknr (rounddown ((len - last_data), +EROFS_BLKSIZ )); +list_add_tail (&e_data->next, extents); + } +return nholes; +} + +int erofs_write_sparse_extents(struct erofs_inode *inode, erofs_off_toff)
lib/inode.c static bool erofs_file_is_compressible(struct
Quoted 34 lines
lib/inode.c +30
-0+{ +struct erofs_sparse_extent_node *e_node; +struct erofs_sparse_extent_iheader *header; +char *buf; +unsigned int p =0 ; +int ret; + + buf =malloc (inode->sparse_extent_isize); +if (buf ==NULL ) +return -ENOMEM ; + header = (struct erofs_sparse_extent_iheader *) buf; + header->count =0 ; + p +=sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { +const struct erofs_sparse_extent ee = { + .ee_lblk =cpu_to_le32 (e_node->lblk), + .ee_pblk =cpu_to_le32 (e_node->pblk), + .ee_len =cpu_to_le32 (e_node->len) + }; +memcpy (buf + p, &ee,sizeof (struct erofs_sparse_extent )); + p +=sizeof (struct erofs_sparse_extent ); + header->count++; +list_del (&e_node->next); +free (e_node); + } + ret =dev_write (buf, off, inode->sparse_extent_isize); +free (buf); +return ret; +} +void erofs_inode_manager_init (void ) {unsigned int i;@@ -304,8 +395,9 @@ static bool erofs_file_is_compressible(struct
erofs_inode*inode)Quoted 31 lines
lib/inode.c +4
-13int erofs_write_file (struct erofs_inode *inode) { -unsigned int nblocks, i; +unsigned int nblocks, i, j, nholes;int ret, fd; +struct erofs_sparse_extent_node *e_node;if (!inode->i_size) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;@@ -322,31 +414,42 @@ int erofs_write_file(struct erofs_inode *inode)/* fallback to all data uncompressed */ inode->datalayout =EROFS_INODE_FLAT_INLINE ; nblocks = inode->i_size /EROFS_BLKSIZ ; - - ret =__allocate_inode_bh_data (inode, nblocks); -if (ret) -return ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 )return -errno; - -for (i =0 ; i < nblocks; ++i) { -char buf[EROFS_BLKSIZ ]; - - ret =read (fd, buf,EROFS_BLKSIZ ); -if (ret !=EROFS_BLKSIZ ) { -if (ret <0 ) + nholes =erofs_read_sparse_extents (fd, &inode->i_sparse_extents); +if (nholes <0 ) {lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 431
Quoted 10 lines
lib/inode.c +10
-0+close (fd); +return nholes; + } + ret =__allocate_inode_bh_data (inode, nblocks - nholes); +if (ret) { +close (fd); +return ret; + } + i = inode->u.i_blkaddr; + inode->sparse_extent_isize = sizeof(struct
erofs_sparse_extent_iheader);lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 433
Quoted 2 lines
lib/inode.c +2
-0+list_for_each_entry (e_node, &inode->i_sparse_extents, next) { + inode->sparse_extent_isize += sizeof(struct
erofs_sparse_extent);lib/inode.c static bool erofs_bh_flush_write_inode(struct
Quoted 30 lines
lib/inode.c +16
-7+ e_node->pblk = i; + ret =lseek (fd,blknr_to_addr (e_node->lblk),SEEK_SET ); +if (ret <0 ) + goto fail; +for (j =0 ; j < e_node->len; j++) { +char buf[EROFS_BLKSIZ ]; + ret =read (fd, buf,EROFS_BLKSIZ ); +if (ret !=EROFS_BLKSIZ ) { +if (ret <0 ) + goto fail; +close (fd); +return -EAGAIN ; + } + ret =blk_write (buf, e_node->pblk + j,1 ); +if (ret) goto fail; -close (fd); -return -EAGAIN ; } - - ret =blk_write (buf, inode->u.i_blkaddr + i,1 ); -if (ret) - goto fail; + i += e_node->len; } -/* read the tail-end data */ inode->idata_size = inode->i_size %EROFS_BLKSIZ ;if (inode->idata_size) {@@ -479,8 +582,14 @@ static bool erofs_bh_flush_write_inode(struct
erofs_buffer_head*bh)lib/inode.c struct erofs_inode
Quoted 28 lines
lib/inode.c +8
-1if (ret)return false;free (inode->compressmeta); + off += inode->extent_isize; } +if (inode->sparse_extent_isize) { + ret =erofs_write_sparse_extents (inode, off); +if (ret) +return false; + } inode->bh =NULL ;erofs_iput (inode);return erofs_bh_flush_generic_end (bh);@@ -737,10 +846,11 @@ struct erofs_inode *erofs_new_inode(void)init_list_head (&inode->i_subdirs);init_list_head (&inode->i_xattrs); +init_list_head (&inode->i_sparse_extents); inode->idata_size =0 ; inode->xattr_isize =0 ; - inode->extent_isize =0 ; + inode->sparse_extent_isize =0 ; inode->bh = inode->bh_inline = inode->bh_data =NULL ; inode->idata =NULL ;@@ -961,4 +1071,3 @@ struct erofs_inode*erofs_mkfs_build_tree_from_path(struct
erofs_inode*parent,Quoted 7 lines
lib/inode.c +0
-2return erofs_mkfs_build_tree (inode); } - --2.9.3
Quoted 9 lines · Pratik Shinde
Hi Gao,
You are correct. The macro logic can be simplified. I will do that. I will work on the kernel part of this change & do some testing on it. I will keep you posted about the change and relevant tests I am running on it.
--Pratik.
Okay, thanks for your effort. :)
Thanks,
Gao Xiang
Quoted 338 lines · Gao Xiang
Quoted 327 lines
Hi Pratik,
Quoted 7 lines · Pratik Shinde
1)Moved on-disk structures to
erofs_fs.h
2)Some naming changes.I think we can keep '
IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only ininode.cso it can be kept there.
what do you think ?What I'm little concerned is the relationship between the name of
IS_HOLEand its implementation...In other words, why
roundup(start, EROFS_BLKSIZ)== start &&
roundup(end, EROFS_BLKSIZ)== end &&
(end - start) %EROFS_BLKSIZ== 0
should be an erofs hole...But that is minor, I reserve my opinion on this for now...
This patch generally looks good to me, yet I haven't played with it till now.
Could you implement a workable RFC patch for kernel side as well? Since I'm still busying in XZ library and other convert patches for 5.6...
I'd like to hear if some other opinions from Chao and Guifu. Since it enhances the on-disk format, we need to think it over (especially its future expandability).
To Chao and Guifu,
Could you have some extra time looking at this stuff as well?Thanks,
Gao XiangQuoted 6 lines
Signed-off-by Pratik Shinde <[email protected]>--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153++++++++++++++++++++++++++++++++++++++++-------
include/erofs_fs.h static inline void
Quoted 40 lines
3 files changed, 150 insertions(+), 23 deletions(-)include/erofs/internal.h +8
-1@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;@@ -93,6 +93,7 @@ struct erofs_inode {unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(int err) return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif include/erofs_fs.h +0
-0@@ -321,5 +321,16 @@ static inline void
erofs_check_ondisk_layout_definitions(void)lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >>
Quoted 21 lines
include/erofs_fs.h +11
-0Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif lib/inode.c +0
-0@@ -38,6 +38,97 @@ static unsigned char erofs_type_by_mode[S_IFMT >>
S_SHIFT] = {lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 5 lines
lib/inode.c +2
-0struct list_head inode_hashtable[NR_INODE_HASHTABLE ]; + +#define IS_HOLE (start, end) (roundup(start, EROFS_BLKSIZ) == start &&\lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 9 lines
lib/inode.c +9
-0+ roundup(end, EROFS_BLKSIZ) == end && \ + (end - start) % EROFS_BLKSIZ == 0) + +/** +read extents of the given file. +record the data extents and link them into a chain. +exclude holes present in file. +*/ +unsigned int erofs_read_sparse_extents(int fd, struct list_head*extents)
Quoted 40 lines
lib/inode.c +40
-0+{ +erofs_blk_t startblk, endblk, datablk; +unsigned int nholes =0 ; +erofs_off_t data, hole, len =0 , last_data; +struct erofs_sparse_extent_node *e_data; + + len =lseek (fd,0 ,SEEK_END ); +if (len <0 ) +return -errno; +if (lseek (fd,0 ,SEEK_SET ) ==-1 ) +return -errno; + data =0 ; + last_data =0 ; +while (data < len) { + hole =lseek (fd, data,SEEK_HOLE ); +if (hole == len) +break ; + data =lseek (fd, hole,SEEK_DATA ); +if (data <0 || hole > data) +return -EINVAL ; +if (IS_HOLE (hole, data)) { + startblk =erofs_blknr (hole); + datablk =erofs_blknr (last_data); + endblk =erofs_blknr (data); + last_data = data; + e_data =malloc (sizeof ( +struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ; + e_data->lblk = datablk; + e_data->len = (startblk - datablk); +list_add_tail (&e_data->next, extents); + nholes += (endblk - startblk); + } + } +/* rounddown to exclude tail-end data */ +if (last_data < len && (len - last_data) >=EROFS_BLKSIZ ) { + e_data =malloc (sizeof (struct erofs_sparse_extent_node )); +if (e_data ==NULL ) +return -ENOMEM ;lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 10 lines
lib/inode.c +10
-0+ startblk =erofs_blknr (last_data); + e_data->lblk = startblk; + e_data->len =erofs_blknr (rounddown ((len - last_data), +EROFS_BLKSIZ )); +list_add_tail (&e_data->next, extents); + } +return nholes; +} + +int erofs_write_sparse_extents(struct erofs_inode *inode, erofs_off_toff)
lib/inode.c static bool erofs_file_is_compressible(struct
Quoted 34 lines
lib/inode.c +30
-0+{ +struct erofs_sparse_extent_node *e_node; +struct erofs_sparse_extent_iheader *header; +char *buf; +unsigned int p =0 ; +int ret; + + buf =malloc (inode->sparse_extent_isize); +if (buf ==NULL ) +return -ENOMEM ; + header = (struct erofs_sparse_extent_iheader *) buf; + header->count =0 ; + p +=sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { +const struct erofs_sparse_extent ee = { + .ee_lblk =cpu_to_le32 (e_node->lblk), + .ee_pblk =cpu_to_le32 (e_node->pblk), + .ee_len =cpu_to_le32 (e_node->len) + }; +memcpy (buf + p, &ee,sizeof (struct erofs_sparse_extent )); + p +=sizeof (struct erofs_sparse_extent ); + header->count++; +list_del (&e_node->next); +free (e_node); + } + ret =dev_write (buf, off, inode->sparse_extent_isize); +free (buf); +return ret; +} +void erofs_inode_manager_init (void ) {unsigned int i;@@ -304,8 +395,9 @@ static bool erofs_file_is_compressible(struct
erofs_inode*inode)Quoted 31 lines
lib/inode.c +4
-13int erofs_write_file (struct erofs_inode *inode) { -unsigned int nblocks, i; +unsigned int nblocks, i, j, nholes;int ret, fd; +struct erofs_sparse_extent_node *e_node;if (!inode->i_size) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;@@ -322,31 +414,42 @@ int erofs_write_file(struct erofs_inode *inode)/* fallback to all data uncompressed */ inode->datalayout =EROFS_INODE_FLAT_INLINE ; nblocks = inode->i_size /EROFS_BLKSIZ ; - - ret =__allocate_inode_bh_data (inode, nblocks); -if (ret) -return ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 )return -errno; - -for (i =0 ; i < nblocks; ++i) { -char buf[EROFS_BLKSIZ ]; - - ret =read (fd, buf,EROFS_BLKSIZ ); -if (ret !=EROFS_BLKSIZ ) { -if (ret <0 ) + nholes =erofs_read_sparse_extents (fd, &inode->i_sparse_extents); +if (nholes <0 ) {lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 431
Quoted 10 lines
lib/inode.c +10
-0+close (fd); +return nholes; + } + ret =__allocate_inode_bh_data (inode, nblocks - nholes); +if (ret) { +close (fd); +return ret; + } + i = inode->u.i_blkaddr; + inode->sparse_extent_isize = sizeof(struct
erofs_sparse_extent_iheader);lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 433
Quoted 2 lines
lib/inode.c +2
-0+list_for_each_entry (e_node, &inode->i_sparse_extents, next) { + inode->sparse_extent_isize += sizeof(struct
erofs_sparse_extent);lib/inode.c static bool erofs_bh_flush_write_inode(struct
Quoted 30 lines
lib/inode.c +16
-7+ e_node->pblk = i; + ret =lseek (fd,blknr_to_addr (e_node->lblk),SEEK_SET ); +if (ret <0 ) + goto fail; +for (j =0 ; j < e_node->len; j++) { +char buf[EROFS_BLKSIZ ]; + ret =read (fd, buf,EROFS_BLKSIZ ); +if (ret !=EROFS_BLKSIZ ) { +if (ret <0 ) + goto fail; +close (fd); +return -EAGAIN ; + } + ret =blk_write (buf, e_node->pblk + j,1 ); +if (ret) goto fail; -close (fd); -return -EAGAIN ; } - - ret =blk_write (buf, inode->u.i_blkaddr + i,1 ); -if (ret) - goto fail; + i += e_node->len; } -/* read the tail-end data */ inode->idata_size = inode->i_size %EROFS_BLKSIZ ;if (inode->idata_size) {@@ -479,8 +582,14 @@ static bool erofs_bh_flush_write_inode(struct
erofs_buffer_head*bh)lib/inode.c struct erofs_inode
Quoted 28 lines
lib/inode.c +8
-1if (ret)return false;free (inode->compressmeta); + off += inode->extent_isize; } +if (inode->sparse_extent_isize) { + ret =erofs_write_sparse_extents (inode, off); +if (ret) +return false; + } inode->bh =NULL ;erofs_iput (inode);return erofs_bh_flush_generic_end (bh);@@ -737,10 +846,11 @@ struct erofs_inode *erofs_new_inode(void)init_list_head (&inode->i_subdirs);init_list_head (&inode->i_xattrs); +init_list_head (&inode->i_sparse_extents); inode->idata_size =0 ; inode->xattr_isize =0 ; - inode->extent_isize =0 ; + inode->sparse_extent_isize =0 ; inode->bh = inode->bh_inline = inode->bh_data =NULL ; inode->idata =NULL ;@@ -961,4 +1071,3 @@ struct erofs_inode*erofs_mkfs_build_tree_from_path(struct
erofs_inode*parent,Quoted 7 lines
lib/inode.c +0
-2return erofs_mkfs_build_tree (inode); } - --2.9.3
Hello Gao,
I have started working on the on kernel side RFC patch of this change , is this still on erofs roadmap ? Will you able to review it?
--Pratik.
Quoted 364 lines · Gao Xiang
Quoted 5 lines · Pratik Shinde
Hi Gao,
You are correct. The macro logic can be simplified. I will do that. I will work on the kernel part of this change & do some testing on it. I will keep you posted about the change and relevant tests I am running
on
Quoted 4 lines
it.
--Pratik.
Okay, thanks for your effort. :)
Thanks,
Gao XiangQuoted 77 lines · Gao Xiang
Quoted 66 lines
Hi Pratik,
Quoted 7 lines · Pratik Shinde
1)Moved on-disk structures to
erofs_fs.h
2)Some naming changes.I think we can keep '
IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only ininode.cso it can be kept there.
what do you think ?What I'm little concerned is the relationship between the name of
IS_HOLEand its implementation...In other words, why
roundup(start, EROFS_BLKSIZ)== start &&
roundup(end, EROFS_BLKSIZ)== end &&
(end - start) %EROFS_BLKSIZ== 0
should be an erofs hole...But that is minor, I reserve my opinion on this for now...
This patch generally looks good to me, yet I haven't played with it till now.
Could you implement a workable RFC patch for kernel side as well? Since I'm still busying in XZ library and other convert patches for 5.6...
I'd like to hear if some other opinions from Chao and Guifu. Since it enhances the on-disk format, we need to think it over (especially its future expandability).
To Chao and Guifu,
Could you have some extra time looking at this stuff as well?Thanks,
Gao XiangQuoted 6 lines
Signed-off-by Pratik Shinde <[email protected]>--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153++++++++++++++++++++++++++++++++++++++++-------
Quoted 17 lines
3 files changed, 150 insertions(+), 23 deletions(-)include/erofs/internal.h +1
-1@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;@@ -93,6 +93,7 @@ struct erofs_inode {Quoted 7 lines
Quoted 7 lines
include/erofs/internal.h static inline const char *erofs_strerror(int
Quoted 7 lines
include/erofs/internal.h +1
-0unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(interr)
Quoted 40 lines
Quoted 40 lines
include/erofs_fs.h static inline void
Quoted 16 lines
include/erofs/internal.h +6
-0return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif include/erofs_fs.h +0
-0@@ -321,5 +321,16 @@ static inline void
erofs_check_ondisk_layout_definitions(void)lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >>
Quoted 21 lines
include/erofs_fs.h +11
-0Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif lib/inode.c +0
-0@@ -38,6 +38,97 @@ static unsigned char erofs_type_by_mode[S_IFMT >>
S_SHIFT] = {Quoted 1 line
Quoted 4 lines
Quoted 4 lines
lib/inode.c
Quoted 4 lines
struct
list_headinode_hashtable[NR_INODE_HASHTABLE];lib/inode.c +2
-0+ +#define IS_HOLE (start, end) (roundup(start, EROFS_BLKSIZ) == start&&
Quoted 2 lines
Quoted 2 lines
\Quoted 1 line
+
roundup(end, EROFS_BLKSIZ)== end &&\
Quoted 28 lines
Quoted 28 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ (end - start) % EROFS_BLKSIZ == 0) + +/** +read extents of the given file. +record the data extents and link them into a chain. +exclude holes present in file. +*/ +unsigned int erofs_read_sparse_extents(int fd, struct list_head*extents)
Quoted 19 lines
lib/inode.c +19
-0+{ +erofs_blk_t startblk, endblk, datablk; +unsigned int nholes =0 ; +erofs_off_t data, hole, len =0 , last_data; +struct erofs_sparse_extent_node *e_data; + + len =lseek (fd,0 ,SEEK_END ); +if (len <0 ) +return -errno; +if (lseek (fd,0 ,SEEK_SET ) ==-1 ) +return -errno; + data =0 ; + last_data =0 ; +while (data < len) { + hole =lseek (fd, data,SEEK_HOLE ); +if (hole == len) +break ; + data =lseek (fd, hole,SEEK_DATA ); +if (data <0 || hole > data)Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+return -EINVAL ; +if (IS_HOLE (hole, data)) { + startblk =erofs_blknr (hole); + datablk =erofs_blknr (last_data); + endblk =erofs_blknr (data); + last_data = data; + e_data = malloc(sizeof( + struct
erofs_sparse_extent_node));Quoted 11 lines
Quoted 11 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 11 lines
lib/inode.c +11
-0+if (e_data ==NULL ) +return -ENOMEM ; + e_data->lblk = datablk; + e_data->len = (startblk - datablk); +list_add_tail (&e_data->next, extents); + nholes += (endblk - startblk); + } + } +/* rounddown to exclude tail-end data */ +if (last_data < len && (len - last_data) >=EROFS_BLKSIZ ) { + e_data = malloc(sizeof(struct
erofs_sparse_extent_node));Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/inode.c +4
-0+if (e_data ==NULL ) +return -ENOMEM ; + startblk =erofs_blknr (last_data); + e_data->lblk = startblk;Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ e_data->len =erofs_blknr (rounddown ((len - last_data), +EROFS_BLKSIZ )); +list_add_tail (&e_data->next, extents); + } +return nholes; +} + +int erofs_write_sparse_extents(struct erofs_inode *inode,
erofs_off_tQuoted 13 lines
Quoted 13 lines
off)
Quoted 12 lines
lib/inode.c +12
-0+{ +struct erofs_sparse_extent_node *e_node; +struct erofs_sparse_extent_iheader *header; +char *buf; +unsigned int p =0 ; +int ret; + + buf =malloc (inode->sparse_extent_isize); +if (buf ==NULL ) +return -ENOMEM ; + header = (struct erofs_sparse_extent_iheader *) buf; + header->count =0 ;Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ p +=sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { +const struct erofs_sparse_extent ee = { + .ee_lblk =cpu_to_le32 (e_node->lblk), + .ee_pblk =cpu_to_le32 (e_node->pblk), + .ee_len =cpu_to_le32 (e_node->len) + }; + memcpy(buf + p, &ee, sizeof(struct
erofs_sparse_extent));Quoted 37 lines
Quoted 37 lines
lib/inode.c static bool erofs_file_is_compressible(struct
Quoted 14 lines
lib/inode.c +10
-0+ p +=sizeof (struct erofs_sparse_extent ); + header->count++; +list_del (&e_node->next); +free (e_node); + } + ret =dev_write (buf, off, inode->sparse_extent_isize); +free (buf); +return ret; +} +void erofs_inode_manager_init (void ) {unsigned int i;@@ -304,8 +395,9 @@ static bool erofs_file_is_compressible(struct
erofs_inode*inode)Quoted 22 lines
lib/inode.c +2
-6int erofs_write_file (struct erofs_inode *inode) { -unsigned int nblocks, i; +unsigned int nblocks, i, j, nholes;int ret, fd; +struct erofs_sparse_extent_node *e_node;if (!inode->i_size) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;@@ -322,31 +414,42 @@ int erofs_write_file(struct erofs_inode *inode)/* fallback to all data uncompressed */ inode->datalayout =EROFS_INODE_FLAT_INLINE ; nblocks = inode->i_size /EROFS_BLKSIZ ; - - ret =__allocate_inode_bh_data (inode, nblocks); -if (ret) -return ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 )return -errno;Quoted 8 lines
Quoted 8 lines
lib/inode.c int erofs_write_file(struct erofs_inode *inode)
Quoted 8 lines
lib/inode.c +1
-7- -for (i =0 ; i < nblocks; ++i) { -char buf[EROFS_BLKSIZ ]; - - ret =read (fd, buf,EROFS_BLKSIZ ); -if (ret !=EROFS_BLKSIZ ) { -if (ret <0 ) + nholes = erofs_read_sparse_extents(fd,
&inode->i_sparse_extents);Quoted 83 lines
Quoted 83 lines
lib/inode.c int erofs_write_file(struct erofs_inode *inode)
Quoted 11 lines
lib/inode.c +11
-0+if (nholes <0 ) { +close (fd); +return nholes; + } + ret =__allocate_inode_bh_data (inode, nblocks - nholes); +if (ret) { +close (fd); +return ret; + } + i = inode->u.i_blkaddr; + inode->sparse_extent_isize = sizeof(struct
erofs_sparse_extent_iheader);lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 433
Quoted 2 lines
lib/inode.c +2
-0+list_for_each_entry (e_node, &inode->i_sparse_extents, next) { + inode->sparse_extent_isize += sizeof(struct
erofs_sparse_extent);lib/inode.c static bool erofs_bh_flush_write_inode(struct
Quoted 30 lines
lib/inode.c +16
-7+ e_node->pblk = i; + ret =lseek (fd,blknr_to_addr (e_node->lblk),SEEK_SET ); +if (ret <0 ) + goto fail; +for (j =0 ; j < e_node->len; j++) { +char buf[EROFS_BLKSIZ ]; + ret =read (fd, buf,EROFS_BLKSIZ ); +if (ret !=EROFS_BLKSIZ ) { +if (ret <0 ) + goto fail; +close (fd); +return -EAGAIN ; + } + ret =blk_write (buf, e_node->pblk + j,1 ); +if (ret) goto fail; -close (fd); -return -EAGAIN ; } - - ret =blk_write (buf, inode->u.i_blkaddr + i,1 ); -if (ret) - goto fail; + i += e_node->len; } -/* read the tail-end data */ inode->idata_size = inode->i_size %EROFS_BLKSIZ ;if (inode->idata_size) {@@ -479,8 +582,14 @@ static bool erofs_bh_flush_write_inode(struct
erofs_buffer_head*bh)lib/inode.c struct erofs_inode
Quoted 28 lines
lib/inode.c +8
-1if (ret)return false;free (inode->compressmeta); + off += inode->extent_isize; } +if (inode->sparse_extent_isize) { + ret =erofs_write_sparse_extents (inode, off); +if (ret) +return false; + } inode->bh =NULL ;erofs_iput (inode);return erofs_bh_flush_generic_end (bh);@@ -737,10 +846,11 @@ struct erofs_inode *erofs_new_inode(void)init_list_head (&inode->i_subdirs);init_list_head (&inode->i_xattrs); +init_list_head (&inode->i_sparse_extents); inode->idata_size =0 ; inode->xattr_isize =0 ; - inode->extent_isize =0 ; + inode->sparse_extent_isize =0 ; inode->bh = inode->bh_inline = inode->bh_data =NULL ; inode->idata =NULL ;@@ -961,4 +1071,3 @@ struct erofs_inode*erofs_mkfs_build_tree_from_path(struct
erofs_inode*parent,Quoted 7 lines
lib/inode.c +0
-2return erofs_mkfs_build_tree (inode); } - --2.9.3
Hi Pratik,
Quoted 4 lines · Pratik Shinde
Hello Gao,
I have started working on the on kernel side RFC patch of this change , is this still on erofs roadmap ? Will you able to review it?
Yes, of course. You could send out when it gets ready. All chinese people are spending Chinese new year festival in hometown, as well as taking action to keep away from Wuhan pneumonia now.
Thanks,
Gao Xiang
Quoted 369 lines
--Pratik.
Quoted 364 lines · Gao Xiang
Quoted 5 lines · Pratik Shinde
Hi Gao,
You are correct. The macro logic can be simplified. I will do that. I will work on the kernel part of this change & do some testing on it. I will keep you posted about the change and relevant tests I am running
on
Quoted 4 lines
it.
--Pratik.
Okay, thanks for your effort. :)
Thanks,
Gao XiangQuoted 77 lines · Gao Xiang
Quoted 66 lines
Hi Pratik,
Quoted 7 lines · Pratik Shinde
1)Moved on-disk structures to
erofs_fs.h
2)Some naming changes.I think we can keep '
IS_HOLE()' macro, otherwise the code does not look clean(if used directly/without macro).Its getting used only ininode.cso it can be kept there.
what do you think ?What I'm little concerned is the relationship between the name of
IS_HOLEand its implementation...In other words, why
roundup(start, EROFS_BLKSIZ)== start &&
roundup(end, EROFS_BLKSIZ)== end &&
(end - start) %EROFS_BLKSIZ== 0
should be an erofs hole...But that is minor, I reserve my opinion on this for now...
This patch generally looks good to me, yet I haven't played with it till now.
Could you implement a workable RFC patch for kernel side as well? Since I'm still busying in XZ library and other convert patches for 5.6...
I'd like to hear if some other opinions from Chao and Guifu. Since it enhances the on-disk format, we need to think it over (especially its future expandability).
To Chao and Guifu,
Could you have some extra time looking at this stuff as well?Thanks,
Gao XiangQuoted 6 lines
Signed-off-by Pratik Shinde <[email protected]>--- include/erofs/internal.h | 9 ++- include/erofs_fs.h | 11 ++++ lib/inode.c | 153++++++++++++++++++++++++++++++++++++++++-------
Quoted 17 lines
3 files changed, 150 insertions(+), 23 deletions(-)include/erofs/internal.h +1
-1@@ -63,7 +63,7 @@ struct erofs_sb_info {extern struct erofs_sb_info sbi;struct erofs_inode { -struct list_head i_hash, i_subdirs, i_xattrs; +struct list_head i_hash, i_subdirs, i_xattrs, i_sparse_extents;unsigned int i_count;struct erofs_inode *i_parent;@@ -93,6 +93,7 @@ struct erofs_inode {Quoted 7 lines
Quoted 7 lines
include/erofs/internal.h static inline const char *erofs_strerror(int
Quoted 7 lines
include/erofs/internal.h +1
-0unsigned int xattr_isize;unsigned int extent_isize; +unsigned int sparse_extent_isize;erofs_nid_t nid;struct erofs_buffer_head *bh;@@ -139,5 +140,11 @@ static inline const char *erofs_strerror(interr)
Quoted 40 lines
Quoted 40 lines
include/erofs_fs.h static inline void
Quoted 16 lines
include/erofs/internal.h +6
-0return msg; } +struct erofs_sparse_extent_node { +struct list_head next; +erofs_blk_t lblk; +erofs_blk_t pblk; +u32 len; +};#endif include/erofs_fs.h +0
-0@@ -321,5 +321,16 @@ static inline void
erofs_check_ondisk_layout_definitions(void)lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >>
Quoted 21 lines
include/erofs_fs.h +11
-0Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); } +/* on-disk sparse extent format */ +struct erofs_sparse_extent { +__le32 ee_lblk; +__le32 ee_pblk; +__le32 ee_len; +}; + +struct erofs_sparse_extent_iheader { +u32 count; +}; +#endif lib/inode.c +0
-0@@ -38,6 +38,97 @@ static unsigned char erofs_type_by_mode[S_IFMT >>
S_SHIFT] = {Quoted 1 line
Quoted 4 lines
Quoted 4 lines
lib/inode.c
Quoted 4 lines
struct
list_headinode_hashtable[NR_INODE_HASHTABLE];lib/inode.c +2
-0+ +#define IS_HOLE (start, end) (roundup(start, EROFS_BLKSIZ) == start&&
Quoted 2 lines
Quoted 2 lines
\Quoted 1 line
+
roundup(end, EROFS_BLKSIZ)== end &&\
Quoted 28 lines
Quoted 28 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ (end - start) % EROFS_BLKSIZ == 0) + +/** +read extents of the given file. +record the data extents and link them into a chain. +exclude holes present in file. +*/ +unsigned int erofs_read_sparse_extents(int fd, struct list_head*extents)
Quoted 19 lines
lib/inode.c +19
-0+{ +erofs_blk_t startblk, endblk, datablk; +unsigned int nholes =0 ; +erofs_off_t data, hole, len =0 , last_data; +struct erofs_sparse_extent_node *e_data; + + len =lseek (fd,0 ,SEEK_END ); +if (len <0 ) +return -errno; +if (lseek (fd,0 ,SEEK_SET ) ==-1 ) +return -errno; + data =0 ; + last_data =0 ; +while (data < len) { + hole =lseek (fd, data,SEEK_HOLE ); +if (hole == len) +break ; + data =lseek (fd, hole,SEEK_DATA ); +if (data <0 || hole > data)Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+return -EINVAL ; +if (IS_HOLE (hole, data)) { + startblk =erofs_blknr (hole); + datablk =erofs_blknr (last_data); + endblk =erofs_blknr (data); + last_data = data; + e_data = malloc(sizeof( + struct
erofs_sparse_extent_node));Quoted 11 lines
Quoted 11 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 11 lines
lib/inode.c +11
-0+if (e_data ==NULL ) +return -ENOMEM ; + e_data->lblk = datablk; + e_data->len = (startblk - datablk); +list_add_tail (&e_data->next, extents); + nholes += (endblk - startblk); + } + } +/* rounddown to exclude tail-end data */ +if (last_data < len && (len - last_data) >=EROFS_BLKSIZ ) { + e_data = malloc(sizeof(struct
erofs_sparse_extent_node));Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
lib/inode.c +4
-0+if (e_data ==NULL ) +return -ENOMEM ; + startblk =erofs_blknr (last_data); + e_data->lblk = startblk;Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ e_data->len =erofs_blknr (rounddown ((len - last_data), +EROFS_BLKSIZ )); +list_add_tail (&e_data->next, extents); + } +return nholes; +} + +int erofs_write_sparse_extents(struct erofs_inode *inode,
erofs_off_tQuoted 13 lines
Quoted 13 lines
off)
Quoted 12 lines
lib/inode.c +12
-0+{ +struct erofs_sparse_extent_node *e_node; +struct erofs_sparse_extent_iheader *header; +char *buf; +unsigned int p =0 ; +int ret; + + buf =malloc (inode->sparse_extent_isize); +if (buf ==NULL ) +return -ENOMEM ; + header = (struct erofs_sparse_extent_iheader *) buf; + header->count =0 ;Quoted 8 lines
Quoted 8 lines
lib/inode.c static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
Quoted 8 lines
lib/inode.c +8
-0+ p +=sizeof (struct erofs_sparse_extent_iheader ); +list_for_each_entry (e_node, &inode->i_sparse_extents, next) { +const struct erofs_sparse_extent ee = { + .ee_lblk =cpu_to_le32 (e_node->lblk), + .ee_pblk =cpu_to_le32 (e_node->pblk), + .ee_len =cpu_to_le32 (e_node->len) + }; + memcpy(buf + p, &ee, sizeof(struct
erofs_sparse_extent));Quoted 37 lines
Quoted 37 lines
lib/inode.c static bool erofs_file_is_compressible(struct
Quoted 14 lines
lib/inode.c +10
-0+ p +=sizeof (struct erofs_sparse_extent ); + header->count++; +list_del (&e_node->next); +free (e_node); + } + ret =dev_write (buf, off, inode->sparse_extent_isize); +free (buf); +return ret; +} +void erofs_inode_manager_init (void ) {unsigned int i;@@ -304,8 +395,9 @@ static bool erofs_file_is_compressible(struct
erofs_inode*inode)Quoted 22 lines
lib/inode.c +2
-6int erofs_write_file (struct erofs_inode *inode) { -unsigned int nblocks, i; +unsigned int nblocks, i, j, nholes;int ret, fd; +struct erofs_sparse_extent_node *e_node;if (!inode->i_size) { inode->datalayout =EROFS_INODE_FLAT_PLAIN ;@@ -322,31 +414,42 @@ int erofs_write_file(struct erofs_inode *inode)/* fallback to all data uncompressed */ inode->datalayout =EROFS_INODE_FLAT_INLINE ; nblocks = inode->i_size /EROFS_BLKSIZ ; - - ret =__allocate_inode_bh_data (inode, nblocks); -if (ret) -return ret; - fd =open (inode->i_srcpath,O_RDONLY |O_BINARY );if (fd <0 )return -errno;Quoted 8 lines
Quoted 8 lines
lib/inode.c int erofs_write_file(struct erofs_inode *inode)
Quoted 8 lines
lib/inode.c +1
-7- -for (i =0 ; i < nblocks; ++i) { -char buf[EROFS_BLKSIZ ]; - - ret =read (fd, buf,EROFS_BLKSIZ ); -if (ret !=EROFS_BLKSIZ ) { -if (ret <0 ) + nholes = erofs_read_sparse_extents(fd,
&inode->i_sparse_extents);Quoted 83 lines
Quoted 83 lines
lib/inode.c int erofs_write_file(struct erofs_inode *inode)
Quoted 11 lines
lib/inode.c +11
-0+if (nholes <0 ) { +close (fd); +return nholes; + } + ret =__allocate_inode_bh_data (inode, nblocks - nholes); +if (ret) { +close (fd); +return ret; + } + i = inode->u.i_blkaddr; + inode->sparse_extent_isize = sizeof(struct
erofs_sparse_extent_iheader);lib/inode.c int erofs_write_file(struct erofs_inode *inode) · line 433
Quoted 2 lines
lib/inode.c +2
-0+list_for_each_entry (e_node, &inode->i_sparse_extents, next) { + inode->sparse_extent_isize += sizeof(struct
erofs_sparse_extent);lib/inode.c static bool erofs_bh_flush_write_inode(struct
Quoted 30 lines
lib/inode.c +16
-7+ e_node->pblk = i; + ret =lseek (fd,blknr_to_addr (e_node->lblk),SEEK_SET ); +if (ret <0 ) + goto fail; +for (j =0 ; j < e_node->len; j++) { +char buf[EROFS_BLKSIZ ]; + ret =read (fd, buf,EROFS_BLKSIZ ); +if (ret !=EROFS_BLKSIZ ) { +if (ret <0 ) + goto fail; +close (fd); +return -EAGAIN ; + } + ret =blk_write (buf, e_node->pblk + j,1 ); +if (ret) goto fail; -close (fd); -return -EAGAIN ; } - - ret =blk_write (buf, inode->u.i_blkaddr + i,1 ); -if (ret) - goto fail; + i += e_node->len; } -/* read the tail-end data */ inode->idata_size = inode->i_size %EROFS_BLKSIZ ;if (inode->idata_size) {@@ -479,8 +582,14 @@ static bool erofs_bh_flush_write_inode(struct
erofs_buffer_head*bh)lib/inode.c struct erofs_inode
Quoted 28 lines
lib/inode.c +8
-1if (ret)return false;free (inode->compressmeta); + off += inode->extent_isize; } +if (inode->sparse_extent_isize) { + ret =erofs_write_sparse_extents (inode, off); +if (ret) +return false; + } inode->bh =NULL ;erofs_iput (inode);return erofs_bh_flush_generic_end (bh);@@ -737,10 +846,11 @@ struct erofs_inode *erofs_new_inode(void)init_list_head (&inode->i_subdirs);init_list_head (&inode->i_xattrs); +init_list_head (&inode->i_sparse_extents); inode->idata_size =0 ; inode->xattr_isize =0 ; - inode->extent_isize =0 ; + inode->sparse_extent_isize =0 ; inode->bh = inode->bh_inline = inode->bh_data =NULL ; inode->idata =NULL ;@@ -961,4 +1071,3 @@ struct erofs_inode*erofs_mkfs_build_tree_from_path(struct
erofs_inode*parent,Quoted 7 lines
lib/inode.c +0
-2return erofs_mkfs_build_tree (inode); } - --2.9.3