f2fs: add fast path for find_next_{zero}bit
Subsystems (2)
Jaegeuk Kim
maintainerPATCH 1/3 f2fs: add fast path for find_next_{zero}bit
This patch adds checking the first two bits when searching zero or one bits to avoid costly find_next_{zero}bit operations.
--- fs/f2fs/dir.c | 10 +++++----- fs/f2fs/f2fs.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/f2fs/gc.c | 3 ++- fs/f2fs/inline.c | 2 +- fs/f2fs/segment.c | 14 ++++++++------ fs/f2fs/segment.h | 11 ++++++----- 6 files changed, 70 insertions(+), 18 deletions(-)
fs/f2fs/dir.c +5 -5
@@ -480,11 +480,11 @@ int room_for_filename(const void *bitmap, int slots, int max_slots)int bit_start =0 ;int zero_start, zero_end; next: - zero_start =find_next_zero_bit_le (bitmap, max_slots, bit_start); + zero_start =f2fs_find_next_zero_bit_le (bitmap, max_slots, bit_start);if (zero_start >= max_slots)return max_slots; - zero_end =find_next_bit_le (bitmap, max_slots, zero_start); + zero_end =f2fs_find_next_bit_le (bitmap, max_slots, zero_start);if (zero_end - zero_start >= slots)return zero_start;
@@ -724,7 +724,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,clear_bit_le (bit_pos + i, &dentry_blk->dentry_bitmap);/* Let's check and deallocate this dentry page */ - bit_pos =find_next_bit_le (&dentry_blk->dentry_bitmap, + bit_pos =f2fs_find_next_bit_le (&dentry_blk->dentry_bitmap,NR_DENTRY_IN_BLOCK ,0 );kunmap (page);/* kunmap - pair of f2fs_find_entry */
@@ -772,7 +772,7 @@ bool f2fs_empty_dir(struct inode *dir) bit_pos =2 ; else bit_pos =0 ; - bit_pos =find_next_bit_le (&dentry_blk->dentry_bitmap, + bit_pos =f2fs_find_next_bit_le (&dentry_blk->dentry_bitmap,NR_DENTRY_IN_BLOCK , bit_pos);kunmap_atomic (dentry_blk);
@@ -796,7 +796,7 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, bit_pos = ((unsigned long )ctx->pos % d->max);while (bit_pos < d->max) { - bit_pos =find_next_bit_le (d->bitmap, d->max, bit_pos); + bit_pos =f2fs_find_next_bit_le (d->bitmap, d->max, bit_pos);if (bit_pos >= d->max)break ;
fs/f2fs/f2fs.h +48 -0
54 lines not on this page · raw message
fs/f2fs/gc.c +2 -1
@@ -301,7 +301,8 @@ static int get_victim_by_default(struct f2fs_sb_info *sbi,unsigned long cost;unsigned int segno; - segno =find_next_bit (p.dirty_segmap, last_segment, p.offset); + segno =f2fs_find_next_bit (p.dirty_segmap, last_segment, + p.offset);if (segno >= last_segment) {if (sbi->last_victim[p.gc_mode]) { last_segment = sbi->last_victim[p.gc_mode];
fs/f2fs/inline.c +1 -1
@@ -593,7 +593,7 @@ bool f2fs_empty_inline_dir(struct inode *dir)return false; dentry_blk =inline_data_addr (ipage); - bit_pos =find_next_bit_le (&dentry_blk->dentry_bitmap, + bit_pos =f2fs_find_next_bit_le (&dentry_blk->dentry_bitmap,NR_INLINE_DENTRY , bit_pos);
fs/f2fs/segment.c +8 -6
@@ -785,10 +785,11 @@ void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)while (1 ) {int i; - start =find_next_bit (prefree_map,MAIN_SEGS (sbi), end +1 ); + start =f2fs_find_next_bit (prefree_map,MAIN_SEGS (sbi), + end +1 );if (start >=MAIN_SEGS (sbi))break ; - end =find_next_zero_bit (prefree_map,MAIN_SEGS (sbi), + end =f2fs_find_next_zero_bit (prefree_map,MAIN_SEGS (sbi), start +1 );for (i = start; i < end; i++)
@@ -1078,16 +1079,17 @@ static void get_new_segment(struct f2fs_sb_info *sbi,spin_lock (&free_i->segmap_lock);if (!new_sec && ((*newseg +1 ) % sbi->segs_per_sec)) { - segno =find_next_zero_bit (free_i->free_segmap, + segno =f2fs_find_next_zero_bit (free_i->free_segmap, (hint +1 ) * sbi->segs_per_sec, *newseg +1 );if (segno < (hint +1 ) * sbi->segs_per_sec) goto got_it; } find_other_zone: - secno =find_next_zero_bit (free_i->free_secmap,MAIN_SECS (sbi), hint); + secno =f2fs_find_next_zero_bit (free_i->free_secmap,MAIN_SECS (sbi), + hint);if (secno >=MAIN_SECS (sbi)) {if (dir ==ALLOC_RIGHT ) { - secno =find_next_zero_bit (free_i->free_secmap, + secno =f2fs_find_next_zero_bit (free_i->free_secmap,MAIN_SECS (sbi),0 );f2fs_bug_on (sbi, secno >=MAIN_SECS (sbi)); }else {
@@ -1103,7 +1105,7 @@ static void get_new_segment(struct f2fs_sb_info *sbi, left_start--;continue ; } - left_start =find_next_zero_bit (free_i->free_secmap, + left_start =f2fs_find_next_zero_bit (free_i->free_secmap,MAIN_SECS (sbi),0 );f2fs_bug_on (sbi, left_start >=MAIN_SECS (sbi));break ;
fs/f2fs/segment.h +6 -5
@@ -336,7 +336,7 @@ static inline unsigned int find_next_inuse(struct free_segmap_info *free_i, {unsigned int ret;spin_lock (&free_i->segmap_lock); - ret =find_next_bit (free_i->free_segmap, max, segno); + ret =f2fs_find_next_bit (free_i->free_segmap, max, segno);spin_unlock (&free_i->segmap_lock);return ret; }
@@ -352,7 +352,7 @@ static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno)clear_bit (segno, free_i->free_segmap); free_i->free_segments++; - next =find_next_bit (free_i->free_segmap, + next =f2fs_find_next_bit (free_i->free_segmap, start_segno + sbi->segs_per_sec, start_segno);if (next >= start_segno + sbi->segs_per_sec) {clear_bit (secno, free_i->free_secmap);
@@ -384,7 +384,7 @@ static inline void __set_test_and_free(struct f2fs_sb_info *sbi,if (test_and_clear_bit (segno, free_i->free_segmap)) { free_i->free_segments++; - next =find_next_bit (free_i->free_segmap, + next =f2fs_find_next_bit (free_i->free_segmap, start_segno + sbi->segs_per_sec, start_segno);if (next >= start_segno + sbi->segs_per_sec) {if (test_and_clear_bit (secno, free_i->free_secmap))
@@ -607,12 +607,13 @@ static inline void check_block_count(struct f2fs_sb_info *sbi,/* check bitmap with valid block count */ do { if (is_valid) { - next_pos = find_next_zero_bit_le(&raw_sit->valid_map, + next_pos = f2fs_find_next_zero_bit_le( + &raw_sit->valid_map, sbi->blocks_per_seg, cur_pos); valid_blocks += next_pos - cur_pos; } else - next_pos = find_next_bit_le(&raw_sit->valid_map, + next_pos = f2fs_find_next_bit_le(&raw_sit->valid_map, sbi->blocks_per_seg, cur_pos); cur_pos = next_pos;
Signature · 1 line
2.8.3
Jaegeuk Kim
maintainerPATCH 2/3 f2fs: use __set_page_dirty_nobuffers directly
This patch replaces the copied code with original generic function.
--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -1
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION
Signature · 1 line
2.8.3
Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly
Hi Jaegeuk,
Quoted 1 line · Jaegeuk Kim
This patch replaces the copied code with original generic function.
Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.
BTW, does the original patch make memory cgroup functionality problematic?
Thanks,
Quoted 72 lines
Signed-off-by Jaegeuk Kim <[email protected]>--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)fs/f2fs/data.c +0
-29@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;fs/f2fs/f2fs.h +5
-1@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION
Jaegeuk Kim
maintainerRe: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly
Quoted 8 lines · Chao Yu
Hi Jaegeuk,
Quoted 1 line · Jaegeuk Kim
This patch replaces the copied code with original generic function.
Will we plan to do further enhance inside
f2fs_set_page_dirty_nobuffers, if we don't it's better revertfe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap__set_page_dirty_nobuffers.
Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.
Quoted 1 line
BTW, does the original patch make memory cgroup functionality problematic?
I don't think there is a problem, since I just copied __set_page_dirty_buffers() except page_has_buffers' stuffs.
Thank you for pointing this out. :)
Quoted 75 lines
Thanks,
Quoted 72 lines
Signed-off-by Jaegeuk Kim <[email protected]>--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)fs/f2fs/data.c +0
-29@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;fs/f2fs/f2fs.h +5
-1@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION
Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly
Quoted 4 lines · Jaegeuk Kim
Quoted 3 lines · Chao Yu
Hi Jaegeuk,
On 2016/10/21 10:28, Jaegeuk Kim wrote:
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
This patch replaces the copied code with original generic function.
Will we plan to do further enhance inside
f2fs_set_page_dirty_nobuffers, if we don't it's better revertfe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap__set_page_dirty_nobuffers.Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.
Why? isn't __set_page_dirty_nobuffers more fit for f2fs' non-buffer management?
Thanks,
Quoted 86 lines
Quoted 2 lines
BTW, does the original patch make memory cgroup functionality problematic?
I don't think there is a problem, since I just copied
__set_page_dirty_buffers()exceptpage_has_buffers' stuffs.Thank you for pointing this out. :)
Quoted 67 lines
Thanks,
Quoted 64 lines
Signed-off-by Jaegeuk Kim <[email protected]>--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)fs/f2fs/data.c +0
-29@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;fs/f2fs/f2fs.h +5
-0@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);Quoted 8 lines
fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210
Quoted 8 lines
fs/f2fs/f2fs.h +0
-1int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION .
Jaegeuk Kim
maintainerRe: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly
Quoted 7 lines · Chao Yu
Quoted 6 lines · Jaegeuk Kim
Quoted 5 lines · Chao Yu
Hi Jaegeuk,
Quoted 1 line · Jaegeuk Kim
This patch replaces the copied code with original generic function.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Will we plan to do further enhance inside
f2fs_set_page_dirty_nobuffers, if we don't it's better revertfe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap__set_page_dirty_nobuffers.Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.Why? isn't
__set_page_dirty_nobuffersmore fit for f2fs' non-buffer management?
For a while ago, when I tried to improve the performance on pmem, I could hit that __set_page_dirty_buffers() slightly improved the bandwidth comparing to __set_page_dirty_nobuffers().
When referencing the below comment written in __set_page_dirty_nobuffers(), it seems I could get that by adopting "top-down" approach instead of "bottom-up", which avoids lock contention as I guess. I couldn't do deep investigation on it though.
/*
* For address_spaces which do not use buffers. Just tag the page as dirty in
* its radix tree.
*
* This is also used when a single buffer is being dirtied: we want to set the
* page dirty in that case, but not all the buffers. This is a "bottom-up"
* dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. *
* The caller must ensure this doesn't race with truncation. Most will simply
* hold the page lock, but e.g. zap_pte_range() calls with the page mapped and
* the pte lock held, which also locks out truncation.
*/
So, I measured the performance again with fxmark on ramdisk, 8 cores, DWAL, bufferedio case. I got 2683158 works/sec w/ "top-down" over 2512609 works/sec w/ "bottom-up".
Thanks,
Quoted 89 lines
Thanks,
Quoted 86 lines
Quoted 2 lines
BTW, does the original patch make memory cgroup functionality problematic?
I don't think there is a problem, since I just copied
__set_page_dirty_buffers()exceptpage_has_buffers' stuffs.Thank you for pointing this out. :)
Quoted 67 lines
Thanks,
Quoted 64 lines
Signed-off-by Jaegeuk Kim <[email protected]>--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)fs/f2fs/data.c +0
-29@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;fs/f2fs/f2fs.h +5
-0@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);Quoted 8 lines
fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210
Quoted 8 lines
fs/f2fs/f2fs.h +0
-1int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION .
Chao Yu
maintainerRe: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly
Quoted 39 lines · Jaegeuk Kim
Quoted 7 lines · Chao Yu
Quoted 6 lines · Jaegeuk Kim
Quoted 5 lines · Chao Yu
Hi Jaegeuk,
Quoted 1 line · Jaegeuk Kim
This patch replaces the copied code with original generic function.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Will we plan to do further enhance inside
f2fs_set_page_dirty_nobuffers, if we don't it's better revertfe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap__set_page_dirty_nobuffers.Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.Why? isn't
__set_page_dirty_nobuffersmore fit for f2fs' non-buffer management?For a while ago, when I tried to improve the performance on pmem, I could hit that
__set_page_dirty_buffers()slightly improved the bandwidth comparing to__set_page_dirty_nobuffers().When referencing the below comment written in
__set_page_dirty_nobuffers(), it seems I could get that by adopting "top-down" approach instead of "bottom-up", which avoids lock contention as I guess. I couldn't do deep investigation on it though./*
* Foraddress_spaceswhich do not use buffers. Just tag the page as dirty in
* its radix tree.
*
* This is also used when a single buffer is being dirtied: we want to set the
* page dirty in that case, but not all the buffers. This is a "bottom-up"
* dirtying, whereas__set_page_dirty_buffers()is a "top-down" dirtying. *
* The caller must ensure this doesn't race with truncation. Most will simply
* hold the page lock, but e.g.zap_pte_range()calls with the page mapped and
* the pte lock held, which also locks out truncation.
*/
Quoted 3 lines
So, I measured the performance again with fxmark on ramdisk, 8 cores, DWAL, bufferedio case. I got 2683158 works/sec w/ "top-down" over 2512609 works/sec w/ "bottom-up".
Thanks for letting me know the history, f2fs_set_page_dirty_nobuffers and __set_page_dirty_nobuffers are almost the same except f2fs_set_page_dirty_nobuffers tries to grab mapping::private_lock additionally.
Maybe holding the private_lock does help the performance on pmem which I can't explain why it happens now...
Thanks,
Quoted 104 lines
Thanks,
Quoted 89 lines
Thanks,
Quoted 86 lines
Quoted 2 lines
BTW, does the original patch make memory cgroup functionality problematic?
I don't think there is a problem, since I just copied
__set_page_dirty_buffers()exceptpage_has_buffers' stuffs.Thank you for pointing this out. :)
Quoted 67 lines
Thanks,
Quoted 64 lines
Signed-off-by Jaegeuk Kim <[email protected]>--- fs/f2fs/data.c | 29 ----------------------------- fs/f2fs/f2fs.h | 6 +++++- 2 files changed, 5 insertions(+), 30 deletions(-)fs/f2fs/data.c +0
-29@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)return 1 ; } -/* -* This was copied from __set_page_dirty_buffers which gives higher performance -* in very high speed storages. (e.g., pmem) -*/ -void f2fs_set_page_dirty_nobuffers (struct page *page) -{ -struct address_space *mapping = page->mapping; -unsigned long flags; - -if (unlikely (!mapping)) -return ; - -spin_lock (&mapping->private_lock); -lock_page_memcg (page); -SetPageDirty (page); -spin_unlock (&mapping->private_lock); - -spin_lock_irqsave (&mapping->tree_lock, flags); -WARN_ON_ONCE (!PageUptodate (page)); -account_page_dirtied (page, mapping); -radix_tree_tag_set (&mapping->page_tree, -page_index (page),PAGECACHE_TAG_DIRTY ); -spin_unlock_irqrestore (&mapping->tree_lock, flags); -unlock_page_memcg (page); - -__mark_inode_dirty (mapping->host,I_DIRTY_PAGES ); -return ; -} -static int f2fs_set_data_page_dirty (struct page *page) {struct address_space *mapping = page->mapping;fs/f2fs/f2fs.h +5
-0@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,return find_next_bit (addr, size, offset +2 ); } +static inline void f2fs_set_page_dirty_nobuffers (struct page *page) +{ +__set_page_dirty_nobuffers (page); +} +#define get_inode_mode (i) \ ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);Quoted 8 lines
fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210
Quoted 8 lines
fs/f2fs/f2fs.h +0
-1int do_write_data_page (struct f2fs_io_info *);int f2fs_map_blocks (struct inode *,struct f2fs_map_blocks *,int ,int );int f2fs_fiemap (struct inode *inode,struct fiemap_extent_info *,u64 ,u64 ); -void f2fs_set_page_dirty_nobuffers (struct page *);void f2fs_invalidate_page (struct page *,unsigned int ,unsigned int );int f2fs_release_page (struct page *,gfp_t );#ifdef CONFIG_MIGRATION .
------------------------------------------------------------------------------
Developer Access Program for Intel Xeon Phi Processors Access to Intel Xeon Phi processor-based developer platforms. With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
_______________________________________________
Linux-f2fs-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Jaegeuk Kim
maintainerPATCH 3/3 f2fs: remove percpu_count due to performance regression
In reply to Jaegeuk KimThis patch removes percpu_count usage due to performance regression in iozone.
--- fs/f2fs/debug.c | 12 ++++++------ fs/f2fs/f2fs.h | 12 ++++++------ fs/f2fs/super.c | 16 +++++----------- 3 files changed, 17 insertions(+), 23 deletions(-)
fs/f2fs/debug.c +6 -6
@@ -313,17 +313,17 @@ static int stat_show(struct seq_file *s, void *v)seq_printf (s," - Inner Struct Count: tree: %d(%d), node: %d\n" , si->ext_tree, si->zombie_tree, si->ext_node);seq_puts (s,"\nBalancing F2FS Async:\n" ); -seq_printf (s," - inmem: %4lld, wb_bios: %4d\n" , +seq_printf (s," - inmem: %4d, wb_bios: %4d\n" , si->inmem_pages, si->wb_bios); -seq_printf (s," - nodes: %4lld in %4d\n" , +seq_printf (s," - nodes: %4d in %4d\n" , si->ndirty_node, si->node_pages); -seq_printf (s," - dents: %4lld in dirs:%4d (%4d)\n" , +seq_printf (s," - dents: %4d in dirs:%4d (%4d)\n" , si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); -seq_printf (s," - datas: %4lld in files:%4d\n" , +seq_printf (s," - datas: %4d in files:%4d\n" , si->ndirty_data, si->ndirty_files); -seq_printf (s," - meta: %4lld in %4d\n" , +seq_printf (s," - meta: %4d in %4d\n" , si->ndirty_meta, si->meta_pages); -seq_printf (s," - imeta: %4lld\n" , +seq_printf (s," - imeta: %4d\n" , si->ndirty_imeta);seq_printf (s," - NATs: %9d/%9d\n - SITs: %9d/%9d\n" , si->dirty_nats, si->nats, si->dirty_sits, si->sits);
fs/f2fs/f2fs.h +6 -6
@@ -824,7 +824,7 @@ struct f2fs_sb_info {atomic_t nr_wb_bios;/* # of writeback bios */ /* # of pages, see count_type */ -struct percpu_counter nr_pages[NR_COUNT_TYPE ]; +atomic_t nr_pages[NR_COUNT_TYPE ];/* # of allocated blocks */ struct percpu_counter alloc_valid_block_count;
@@ -1238,7 +1238,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,static inline void inc_page_count (struct f2fs_sb_info *sbi,int count_type) { -percpu_counter_inc (&sbi->nr_pages[count_type]); +atomic_inc (&sbi->nr_pages[count_type]);if (count_type ==F2FS_DIRTY_DATA || count_type ==F2FS_INMEM_PAGES )return ;
@@ -1255,7 +1255,7 @@ static inline void inode_inc_dirty_pages(struct inode *inode)static inline void dec_page_count (struct f2fs_sb_info *sbi,int count_type) { -percpu_counter_dec (&sbi->nr_pages[count_type]); +atomic_dec (&sbi->nr_pages[count_type]); }static inline void inode_dec_dirty_pages (struct inode *inode)
@@ -1271,7 +1271,7 @@ static inline void inode_dec_dirty_pages(struct inode *inode)static inline s64 get_pages (struct f2fs_sb_info *sbi,int count_type) { -return percpu_counter_sum_positive (&sbi->nr_pages[count_type]); +return atomic_read (&sbi->nr_pages[count_type]); }static inline s64 get_dirty_pages (struct inode *inode)
@@ -2239,8 +2239,8 @@ struct f2fs_stat_info {unsigned long long hit_largest, hit_cached, hit_rbtree;unsigned long long hit_total, total_ext;int ext_tree, zombie_tree, ext_node; -s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; -s64 inmem_pages; +int ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta; +int inmem_pages;unsigned int ndirty_dirs, ndirty_files, ndirty_all;int nats, dirty_nats, sits, dirty_sits, free_nids, alloc_nids;int total_count, utilization;
fs/f2fs/super.c +5 -11
@@ -696,10 +696,6 @@ static void f2fs_destroy_inode(struct inode *inode)static void destroy_percpu_info (struct f2fs_sb_info *sbi) { -int i; - -for (i =0 ; i <NR_COUNT_TYPE ; i++) -percpu_counter_destroy (&sbi->nr_pages[i]);percpu_counter_destroy (&sbi->alloc_valid_block_count);percpu_counter_destroy (&sbi->total_valid_inode_count); }
@@ -1449,6 +1445,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi)static void init_sb_info (struct f2fs_sb_info *sbi) {struct f2fs_super_block *raw_super = sbi->raw_super; +int i; sbi->log_sectors_per_block =le32_to_cpu (raw_super->log_sectors_per_block);
@@ -1473,6 +1470,9 @@ static void init_sb_info(struct f2fs_sb_info *sbi) sbi->interval_time[REQ_TIME ] =DEF_IDLE_INTERVAL ;clear_sbi_flag (sbi,SBI_NEED_FSCK ); +for (i =0 ; i <NR_COUNT_TYPE ; i++) +atomic_set (&sbi->nr_pages[i],0 ); +INIT_LIST_HEAD (&sbi->s_list);mutex_init (&sbi->umount_mutex);mutex_init (&sbi->wio_mutex[NODE ]);
@@ -1488,13 +1488,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi)static int init_percpu_info (struct f2fs_sb_info *sbi) { -int i, err; - -for (i =0 ; i <NR_COUNT_TYPE ; i++) { - err =percpu_counter_init (&sbi->nr_pages[i],0 ,GFP_KERNEL ); -if (err) -return err; - } +int err; err =percpu_counter_init (&sbi->alloc_valid_block_count,0 ,GFP_KERNEL );if (err)
Signature · 1 line
2.8.3
Quoted 2 lines · Jaegeuk Kim
This patch adds checking the first two bits when searching zero or one bits to avoid costly find_next_{zero}bit operations.
Please fix this in the generic iplementations instead of adding your local workarounds.