erofs: allow readdir() to be interrupted
Chao Yu
maintainerPATCH 1/2 erofs: allow readdir() to be interrupted
In a quick slow device, readdir() may loop for long time in large directory, let's give a chance to allow it to be interrupted by userspace.
--- fs/erofs/dir.c | 7 +++++++ 1 file changed, 7 insertions(+)
fs/erofs/dir.c +7 -0
@@ -58,6 +58,13 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)struct erofs_dirent *de;unsigned int nameoff, maxsize; +/* allow readdir() to be interrupted */ +if (fatal_signal_pending (current)) { + err = -ERESTARTSYS ; +break ; + } +cond_resched (); + de =erofs_bread (&buf, dbstart, true);if (IS_ERR (de)) { erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",
Signature · 1 line
2.49.0
Chao Yu
maintainerPATCH 2/2 erofs: support to readahead dirent blocks in erofs_readdir()
This patch supports to readahead more blocks in erofs_readdir(), it can enhance performance in large direcotry.
readdir test in a large directory which contains 12000 sub-files.
files_per_second
Before: 926385.54
After: 2380435.562
--- fs/erofs/dir.c | 8 ++++++++ fs/erofs/internal.h | 3 +++ 2 files changed, 11 insertions(+)
fs/erofs/dir.c +8 -0
@@ -47,8 +47,10 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)struct inode *dir =file_inode (f);struct erofs_buf buf = __EROFS_BUF_INITIALIZER;struct super_block *sb = dir->i_sb; +struct file_ra_state *ra = &f->f_ra;unsigned long bsz = sb->s_blocksize;unsigned int ofs =erofs_blkoff (sb, ctx->pos); +unsigned long nr_pages =DIV_ROUND_UP_POW2 (dir->i_size,PAGE_SIZE );int err =0 ;bool initial = true;
@@ -65,6 +67,12 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) }cond_resched (); +/* readahead blocks to enhance performance in large directory */ +if (nr_pages - dbstart >1 && !ra_has_index (ra, dbstart)) +page_cache_sync_readahead (dir->i_mapping, ra, f, + dbstart,min (nr_pages - dbstart, + (pgoff_t )MAX_DIR_RA_PAGES )); + de =erofs_bread (&buf, dbstart, true);if (IS_ERR (de)) { erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",
fs/erofs/internal.h +3 -0
@@ -238,6 +238,9 @@ EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) +/* maximum readahead pages of directory */ +#define MAX_DIR_RA_PAGES 4 +struct erofs_inode {erofs_nid_t nid;
Signature · 1 line
2.49.0
Quoted 5 lines · Chao Yu
In a quick slow device,
readdir()may loop for long time in large directory, let's give a chance to allow it to be interrupted by userspace.Signed-off-by Chao Yu <[email protected]>
fs/erofs/dir.c static int erofs_readdir(struct file *f, struct dir_context *ctx) · line 61
Quoted 13 lines
--- fs/erofs/dir.c | 7 +++++++ 1 file changed, 7 insertions(+)fs/erofs/dir.c +1
-0@@ -58,6 +58,13 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)struct erofs_dirent *de;unsigned int nameoff, maxsize; +/* allow readdir() to be interrupted */
Hi Chao,
It seems that comment is unnecessary since the following code is obvious, if you have no objection I will remove this comment when applying.
Thanks,
Gao Xiang
Re: [PATCH 2/2] erofs: support to readahead dirent blocks in erofs_readdir()
In reply to Chao YuHi Chao,
Quoted 48 lines · Chao Yu
This patch supports to readahead more blocks in
erofs_readdir(), it can enhance performance in large direcotry.readdir test in a large directory which contains 12000 sub-files.
files_per_second
Before: 926385.54
After: 2380435.562Signed-off-by Chao Yu <[email protected]>--- fs/erofs/dir.c | 8 ++++++++ fs/erofs/internal.h | 3 +++ 2 files changed, 11 insertions(+)fs/erofs/dir.c +8
-0@@ -47,8 +47,10 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)struct inode *dir =file_inode (f);struct erofs_buf buf = __EROFS_BUF_INITIALIZER;struct super_block *sb = dir->i_sb; +struct file_ra_state *ra = &f->f_ra;unsigned long bsz = sb->s_blocksize;unsigned int ofs =erofs_blkoff (sb, ctx->pos); +unsigned long nr_pages =DIV_ROUND_UP_POW2 (dir->i_size,PAGE_SIZE );int err =0 ;bool initial = true;@@ -65,6 +67,12 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) }cond_resched (); +/* readahead blocks to enhance performance in large directory */ +if (nr_pages - dbstart >1 && !ra_has_index (ra, dbstart)) +page_cache_sync_readahead (dir->i_mapping, ra, f, + dbstart,min (nr_pages - dbstart, + (pgoff_t )MAX_DIR_RA_PAGES )); + de =erofs_bread (&buf, dbstart, true);if (IS_ERR (de)) { erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",fs/erofs/internal.h +0
-0
fs/erofs/internal.h EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) · line 242
Quoted 5 lines
fs/erofs/internal.h +2
-0@@ -238,6 +238,9 @@ EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1)#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) +/* maximum readahead pages of directory */ +#define MAX_DIR_RA_PAGES 4
Could we set it as a per-sb sysfs configuration for users to config?
Thanks,
Gao Xiang
Chao Yu
maintainerQuoted 26 lines · Gao Xiang
Quoted 5 lines · Chao Yu
In a quick slow device,
readdir()may loop for long time in large directory, let's give a chance to allow it to be interrupted by userspace.Signed-off-by Chao Yu <[email protected]>Reviewed-by Gao Xiang <[email protected]>Quoted 11 lines
---
fs/erofs/dir.c| 7 +++++++1 file changed, 7 insertions(+)fs/erofs/dir.c +0
-0@@ -58,6 +58,13 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) struct erofs_dirent *de; unsigned int nameoff, maxsize;Quoted 1 line
+ /* allow
readdir()to be interrupted */Hi Chao,
Quoted 3 lines
It seems that comment is unnecessary since the following code is obvious, if you have no objection I will remove this comment when applying.
Xiang, sure, no problem. :)
Thanks,
Quoted 3 lines
Thanks,
Gao Xiang
Chao Yu
maintainerRe: [PATCH 2/2] erofs: support to readahead dirent blocks in erofs_readdir()
In reply to Gao XiangQuoted 47 lines · Gao Xiang
Hi Chao,
Quoted 44 lines · Chao Yu
This patch supports to readahead more blocks in
erofs_readdir(), it can enhance performance in large direcotry.readdir test in a large directory which contains 12000 sub-files.
files_per_second
Before: 926385.54
After: 2380435.562Signed-off-by Chao Yu <[email protected]>---
fs/erofs/dir.c | 8 ++++++++
fs/erofs/internal.h| 3 +++2 files changed, 11 insertions(+)fs/erofs/dir.c +2
-0@@ -47,8 +47,10 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) struct inode *dir = file_inode(f); struct erofs_buf buf = __EROFS_BUF_INITIALIZER; struct super_block *sb = dir->i_sb; + struct file_ra_state *ra = &f->f_ra; unsigned long bsz = sb->s_blocksize; unsigned int ofs = erofs_blkoff(sb, ctx->pos); + unsigned long nr_pages = DIV_ROUND_UP_POW2(dir->i_size, PAGE_SIZE); int err = 0; bool initial = true; @@ -65,6 +67,12 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)}
cond_resched();
+ /* readahead blocks to enhance performance in large directory */ + if (nr_pages- dbstart > 1 && !ra_has_index(ra, dbstart))
+ page_cache_sync_readahead(dir->i_mapping, ra, f,
+ dbstart, min(nr_pages - dbstart,
+ (pgoff_t)MAX_DIR_RA_PAGES));
+
de =erofs_bread(&buf, dbstart, true);
if (IS_ERR(de)) {
erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",fs/erofs/internal.h +0
-1--- a/fs/erofs/internal.h
Quoted 8 lines
Quoted 6 lines
+++
b/fs/erofs/internal.hfs/erofs/internal.h +0
-0@@ -238,6 +238,9 @@ EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) #define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) #define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)+/* maximum readahead pages of directory */
+#define MAX_DIR_RA_PAGES 4Could we set it as a per-sb sysfs configuration for users to config?
Xiang,
Oh, that will be better, how about introducing new sysfs in separated patch?
Thanks,
Quoted 3 lines
Thanks,
Gao Xiang
Re: [PATCH 2/2] erofs: support to readahead dirent blocks in erofs_readdir()
Quoted 57 lines · Chao Yu
Quoted 47 lines · Gao Xiang
Hi Chao,
Quoted 44 lines · Chao Yu
This patch supports to readahead more blocks in
erofs_readdir(), it can enhance performance in large direcotry.readdir test in a large directory which contains 12000 sub-files.
files_per_second
Before: 926385.54
After: 2380435.562Signed-off-by Chao Yu <[email protected]>--- fs/erofs/dir.c | 8 ++++++++ fs/erofs/internal.h | 3 +++ 2 files changed, 11 insertions(+)fs/erofs/dir.c +2
-0@@ -47,8 +47,10 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) struct inode *dir = file_inode(f); struct erofs_buf buf = __EROFS_BUF_INITIALIZER; struct super_block *sb = dir->i_sb; + struct file_ra_state *ra = &f->f_ra; unsigned long bsz = sb->s_blocksize; unsigned int ofs = erofs_blkoff(sb, ctx->pos); + unsigned long nr_pages = DIV_ROUND_UP_POW2(dir->i_size, PAGE_SIZE); int err = 0; bool initial = true; @@ -65,6 +67,12 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)}
cond_resched();
+ /* readahead blocks to enhance performance in large directory */
+ if (nr_pages- dbstart > 1 && !ra_has_index(ra, dbstart)) + page_cache_sync_readahead(dir->i_mapping, ra, f,
+ dbstart, min(nr_pages - dbstart,
+ (pgoff_t)MAX_DIR_RA_PAGES));fs/erofs/dir.c +1
-0+ de = erofs_bread(&buf, dbstart, true); if (IS_ERR(de)) { erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",fs/erofs/internal.h +0
-1--- a/fs/erofs/internal.hQuoted 8 lines
fs/erofs/internal.h EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) · line 241
Quoted 6 lines
+++
b/fs/erofs/internal.hfs/erofs/internal.h +1
-0@@ -238,6 +238,9 @@ EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) #define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) #define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) +/* maximum readahead pages of directory */ +#define MAX_DIR_RA_PAGES 4Could we set it as a per-sb sysfs configuration for users to config?
Quoted 3 lines
Xiang,
Oh, that will be better, how about introducing new sysfs in separated patch?
Hi Chao,
Thanks for your interest but in that case it could cause some bisect issue anyway, could we just use one patch to add this optimization for slow devices?
Thanks,
Gao Xiang
Quoted 7 lines
Thanks,
Quoted 3 lines
Thanks,
Gao Xiang
Chao Yu
maintainerRe: [PATCH 2/2] erofs: support to readahead dirent blocks in erofs_readdir()
Quoted 62 lines · Gao Xiang
Quoted 59 lines · Chao Yu
Quoted 47 lines · Gao Xiang
Hi Chao,
Quoted 44 lines · Chao Yu
This patch supports to readahead more blocks in
erofs_readdir(), it can enhance performance in large direcotry.readdir test in a large directory which contains 12000 sub-files.
files_per_second
Before: 926385.54
After: 2380435.562Signed-off-by Chao Yu <[email protected]>---
fs/erofs/dir.c | 8 ++++++++
fs/erofs/internal.h| 3 +++2 files changed, 11 insertions(+)fs/erofs/dir.c +2
-0@@ -47,8 +47,10 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) struct inode *dir = file_inode(f); struct erofs_buf buf = __EROFS_BUF_INITIALIZER; struct super_block *sb = dir->i_sb; + struct file_ra_state *ra = &f->f_ra; unsigned long bsz = sb->s_blocksize; unsigned int ofs = erofs_blkoff(sb, ctx->pos); + unsigned long nr_pages = DIV_ROUND_UP_POW2(dir->i_size, PAGE_SIZE); int err = 0; bool initial = true; @@ -65,6 +67,12 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx)}
cond_resched();
+ /* readahead blocks to enhance performance in large directory */ + if (nr_pages- dbstart > 1 && !ra_has_index(ra, dbstart))
+ page_cache_sync_readahead(dir->i_mapping, ra, f,
+ dbstart, min(nr_pages - dbstart,
+ (pgoff_t)MAX_DIR_RA_PAGES));
+
de =erofs_bread(&buf, dbstart, true);
if (IS_ERR(de)) {
erofs_err(sb, "failed to readdir of logical block %llu of nid %llu",fs/erofs/internal.h +0
-1--- a/fs/erofs/internal.hQuoted 8 lines
Quoted 6 lines
+++
b/fs/erofs/internal.hfs/erofs/internal.h +0
-0@@ -238,6 +238,9 @@ EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) #define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) #define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2)+/* maximum readahead pages of directory */
+#define MAX_DIR_RA_PAGES 4Could we set it as a per-sb sysfs configuration for users to config?
Xiang,
Quoted 7 lines
Quoted 1 line
Oh, that will be better, how about introducing new sysfs in separated patch?
Hi Chao,
Thanks for your interest but in that case it could cause some bisect issue anyway, could we just use one patch to add this optimization for slow devices?
Xiang,
Yeah, one patch includes all these looks fine to me, anyway, let me update v2.
Thanks,
Quoted 12 lines
Thanks,
Gao XiangQuoted 7 lines
Thanks,
Quoted 3 lines
Thanks,
Gao Xiang