ceph: fix blindly expanding the readahead windows
PATCH v2 1/1 ceph: fix blindly expanding the readahead windows
From: Xiubo Li <[email protected]>
Blindly expanding the readahead windows will cause unneccessary pagecache thrashing and also will introdue the network workload. We should disable expanding the windows if the readahead is disabled and also shouldn't expand the windows too much.
Expanding forward firstly instead of expanding backward for possible sequential reads.
URL: https://www.spinics.net/lists/ceph-users/msg76183.html
---
V2:
- fix possible cross-block issue pointed out by Ilya.
fs/ceph/addr.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)
fs/ceph/addr.c +18 -6
@@ -188,16 +188,28 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)struct inode *inode = rreq->inode;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_file_layout *lo = &ci->i_layout; +unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; +unsigned long max_len = max_pages <<PAGE_SHIFT ; +unsigned long len;u32 blockoff;u64 blockno; -/* Expand the start downward */ - blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); - rreq->start = blockno * lo->stripe_unit; - rreq->len += blockoff; +/* Readahead is disabled */ +if (!max_pages) +return ; + +/* Try to expand the length forward by rounding up it to the next block */ +div_u64_rem (rreq->start + rreq->len, lo->stripe_unit, &blockoff); + len = lo->stripe_unit - blockoff; +if (rreq->len + len <= max_len) + rreq->len += len; -/* Now, round up the length to the next block */ - rreq->len =roundup (rreq->len, lo->stripe_unit); +/* Try to expand the start downward */ + blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); +if (rreq->len + blockoff <= max_len) { + rreq->start = blockno * lo->stripe_unit; + rreq->len += blockoff; + } }static bool ceph_netfs_clamp_length (struct netfs_io_subrequest *subreq)
Signature · 1 line
2.40.0
Hi Xiubo,
I just send a patch to solve the same issue[1]. My patch bound the request by i_size, which is orthogonal to your changes. To incorporate both, I propose the following patch.
Also, since this is a performance regression, I think we should backport it to stable versions?
Another comment inline.
Hu Weiwen
[1]: https://lore.kernel.org/ceph-devel/[email protected]/
fs/ceph/addr.c +20 -6
@@ -187,16 +187,30 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)struct inode *inode = rreq->inode;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_file_layout *lo = &ci->i_layout; +unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; +unsigned long max_len = max_pages <<PAGE_SHIFT ; +loff_t end = rreq->start + rreq->len, new_end;u32 blockoff;u64 blockno; -/* Expand the start downward */ - blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); - rreq->start = blockno * lo->stripe_unit; - rreq->len += blockoff; +/* Readahead is disabled */ +if (!max_pages) +return ; + +/* Try to expand the length forward by rounding up it to the next block */ + new_end =round_up (end, lo->stripe_unit); +/* But do not exceed the file size, +* unless the original request already exceeds it. */ + new_end =min (new_end, rreq->i_size); +if (new_end > end && new_end <= rreq->start + max_len) + rreq->len = new_end - rreq->start; -/* Now, round up the length to the next block */ - rreq->len =roundup (rreq->len, lo->stripe_unit); +/* Try to expand the start downward */ + blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); +if (rreq->len + blockoff <= max_len) { + rreq->start = blockno * lo->stripe_unit; + rreq->len += blockoff; + } }static bool ceph_netfs_clamp_length (struct netfs_io_subrequest *subreq)
Quoted 36 lines · On Thu, May 04, 2023 at 01:23:06PM +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
Blindly expanding the readahead windows will cause unneccessary pagecache thrashing and also will introdue the network workload. We should disable expanding the windows if the readahead is disabled and also shouldn't expand the windows too much.
Expanding forward firstly instead of expanding backward for possible sequential reads.
URL: https://www.spinics.net/lists/ceph-users/msg76183.html
Signed-off-by Xiubo Li <[email protected]>---
V2:
- fix possible cross-block issue pointed out by Ilya.fs/ceph/addr.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)fs/ceph/addr.c +3
-1@@ -188,16 +188,28 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)struct inode *inode = rreq->inode;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_file_layout *lo = &ci->i_layout; +unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; +unsigned long max_len = max_pages <<PAGE_SHIFT ; +unsigned long len;u32 blockoff;u64 blockno; -/* Expand the start downward */
fs/ceph/addr.c static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) · line 203
Quoted 10 lines
fs/ceph/addr.c +7
-3- blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); - rreq->start = blockno * lo->stripe_unit; - rreq->len += blockoff; +/* Readahead is disabled */ +if (!max_pages) +return ; + +/* Try to expand the length forward by rounding up it to the next block */ +div_u64_rem (rreq->start + rreq->len, lo->stripe_unit, &blockoff); + len = lo->stripe_unit - blockoff;
This would expand the request by a whole block, if the original request is already aligned (blockoff == 0).
Quoted 17 lines
fs/ceph/addr.c +8
-2+if (rreq->len + len <= max_len) + rreq->len += len; -/* Now, round up the length to the next block */ - rreq->len =roundup (rreq->len, lo->stripe_unit); +/* Try to expand the start downward */ + blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); +if (rreq->len + blockoff <= max_len) { + rreq->start = blockno * lo->stripe_unit; + rreq->len += blockoff; + } }static bool ceph_netfs_clamp_length (struct netfs_io_subrequest *subreq)Signature · 1 line
2.40.0
Quoted 94 lines · 胡玮文
Hi Xiubo,
I just send a patch to solve the same issue[1]. My patch bound the request by
i_size, which is orthogonal to your changes. To incorporate both, I propose the following patch.Also, since this is a performance regression, I think we should backport it to stable versions?
Another comment inline.
Hu Weiwen
[1]: https://lore.kernel.org/ceph-devel/[email protected]/
fs/ceph/addr.c +20
-6@@ -187,16 +187,30 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)struct inode *inode = rreq->inode;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_file_layout *lo = &ci->i_layout; +unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; +unsigned long max_len = max_pages <<PAGE_SHIFT ; +loff_t end = rreq->start + rreq->len, new_end;u32 blockoff;u64 blockno; -/* Expand the start downward */ - blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); - rreq->start = blockno * lo->stripe_unit; - rreq->len += blockoff; +/* Readahead is disabled */ +if (!max_pages) +return ; + +/* Try to expand the length forward by rounding up it to the next block */ + new_end =round_up (end, lo->stripe_unit); +/* But do not exceed the file size, +* unless the original request already exceeds it. */ + new_end =min (new_end, rreq->i_size); +if (new_end > end && new_end <= rreq->start + max_len) + rreq->len = new_end - rreq->start; -/* Now, round up the length to the next block */ - rreq->len =roundup (rreq->len, lo->stripe_unit); +/* Try to expand the start downward */ + blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); +if (rreq->len + blockoff <= max_len) { + rreq->start = blockno * lo->stripe_unit; + rreq->len += blockoff; + } }static bool ceph_netfs_clamp_length (struct netfs_io_subrequest *subreq)Quoted 35 lines · On Thu, May 04, 2023 at 01:23:06PM +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
Blindly expanding the readahead windows will cause unneccessary pagecache thrashing and also will introdue the network workload. We should disable expanding the windows if the readahead is disabled and also shouldn't expand the windows too much.
Expanding forward firstly instead of expanding backward for possible sequential reads.
URL: https://www.spinics.net/lists/ceph-users/msg76183.html
Signed-off-by Xiubo Li <[email protected]>---
V2:
- fix possible cross-block issue pointed out by Ilya.fs/ceph/addr.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-)fs/ceph/addr.c +3
-0@@ -188,16 +188,28 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)struct inode *inode = rreq->inode;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_file_layout *lo = &ci->i_layout; +unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; +unsigned long max_len = max_pages <<PAGE_SHIFT ; +unsigned long len;u32 blockoff;u64 blockno;
Quoted 13 lines
fs/ceph/addr.c static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq)
Quoted 11 lines
fs/ceph/addr.c +7
-4-/* Expand the start downward */ - blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); - rreq->start = blockno * lo->stripe_unit; - rreq->len += blockoff; +/* Readahead is disabled */ +if (!max_pages) +return ; + +/* Try to expand the length forward by rounding up it to the next block */ +div_u64_rem (rreq->start + rreq->len, lo->stripe_unit, &blockoff); + len = lo->stripe_unit - blockoff;This would expand the request by a whole block, if the original request is already aligned (blockoff == 0).
Hi 胡玮文
Good catch.
The above patch looks better and I will revise it.
Thanks
- Xiubo
Quoted 17 lines
Quoted 17 lines
fs/ceph/addr.c +8
-2+if (rreq->len + len <= max_len) + rreq->len += len; -/* Now, round up the length to the next block */ - rreq->len =roundup (rreq->len, lo->stripe_unit); +/* Try to expand the start downward */ + blockno =div_u64_rem (rreq->start, lo->stripe_unit, &blockoff); +if (rreq->len + blockoff <= max_len) { + rreq->start = blockno * lo->stripe_unit; + rreq->len += blockoff; + } }static bool ceph_netfs_clamp_length (struct netfs_io_subrequest *subreq)Signature · 1 line
2.40.0