Skip to contentSkip to conversation

f2fs: add fast path for find_next_{zero}bit

PATCH In review Jaegeuk Kim 4 participants 9 messages Oct 21, 2016 – Nov 4, 2016

Subsystems (2)
lore
Share & tools
lore

Jaegeuk Kim

author maintainer
#0 lore raw

PATCH 1/3 f2fs: add fast path for find_next_{zero}bit

This patch adds checking the first two bits when searching zero or one bits to avoid costly find_next_{zero}bit operations.

---
 fs/f2fs/dir.c     | 10 +++++-----
 fs/f2fs/f2fs.h    | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/f2fs/gc.c      |  3 ++-
 fs/f2fs/inline.c  |  2 +-
 fs/f2fs/segment.c | 14 ++++++++------
 fs/f2fs/segment.h | 11 ++++++-----
 6 files changed, 70 insertions(+), 18 deletions(-)
fs/f2fs/dir.c +5 -5
index 5f5678e..f3a4fce 100644
@@ -480,11 +480,11 @@ int room_for_filename(const void *bitmap, int slots, int max_slots)
 	int bit_start = 0;
 	int zero_start, zero_end;
 next:
-	zero_start = find_next_zero_bit_le(bitmap, max_slots, bit_start);
+	zero_start = f2fs_find_next_zero_bit_le(bitmap, max_slots, bit_start);
 	if (zero_start >= max_slots)
 		return max_slots;
 
-	zero_end = find_next_bit_le(bitmap, max_slots, zero_start);
+	zero_end = f2fs_find_next_bit_le(bitmap, max_slots, zero_start);
 	if (zero_end - zero_start >= slots)
 		return zero_start;
 
@@ -724,7 +724,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
 		clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap);
 
 	/* Let's check and deallocate this dentry page */
-	bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
+	bit_pos = f2fs_find_next_bit_le(&dentry_blk->dentry_bitmap,
 			NR_DENTRY_IN_BLOCK,
 			0);
 	kunmap(page); /* kunmap - pair of f2fs_find_entry */
@@ -772,7 +772,7 @@ bool f2fs_empty_dir(struct inode *dir)
 			bit_pos = 2;
 		else
 			bit_pos = 0;
-		bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
+		bit_pos = f2fs_find_next_bit_le(&dentry_blk->dentry_bitmap,
 						NR_DENTRY_IN_BLOCK,
 						bit_pos);
 		kunmap_atomic(dentry_blk);
@@ -796,7 +796,7 @@ bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 	bit_pos = ((unsigned long)ctx->pos % d->max);
 
 	while (bit_pos < d->max) {
-		bit_pos = find_next_bit_le(d->bitmap, d->max, bit_pos);
+		bit_pos = f2fs_find_next_bit_le(d->bitmap, d->max, bit_pos);
 		if (bit_pos >= d->max)
 			break;
 
fs/f2fs/f2fs.h +48 -0

54 lines not on this page · raw message

fs/f2fs/gc.c +2 -1
index 9c18917..f35fca5 100644
@@ -301,7 +301,8 @@ static int get_victim_by_default(struct f2fs_sb_info *sbi,
 		unsigned long cost;
 		unsigned int segno;
 
-		segno = find_next_bit(p.dirty_segmap, last_segment, p.offset);
+		segno = f2fs_find_next_bit(p.dirty_segmap, last_segment,
+								p.offset);
 		if (segno >= last_segment) {
 			if (sbi->last_victim[p.gc_mode]) {
 				last_segment = sbi->last_victim[p.gc_mode];
fs/f2fs/inline.c +1 -1
index 8cab5df..def731a 100644
@@ -593,7 +593,7 @@ bool f2fs_empty_inline_dir(struct inode *dir)
 		return false;
 
 	dentry_blk = inline_data_addr(ipage);
-	bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
+	bit_pos = f2fs_find_next_bit_le(&dentry_blk->dentry_bitmap,
 					NR_INLINE_DENTRY,
 					bit_pos);
 
fs/f2fs/segment.c +8 -6
index bbb9355..0d70155 100644
@@ -785,10 +785,11 @@ void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 
 	while (1) {
 		int i;
-		start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
+		start = f2fs_find_next_bit(prefree_map, MAIN_SEGS(sbi),
+								end + 1);
 		if (start >= MAIN_SEGS(sbi))
 			break;
-		end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
+		end = f2fs_find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
 								start + 1);
 
 		for (i = start; i < end; i++)
@@ -1078,16 +1079,17 @@ static void get_new_segment(struct f2fs_sb_info *sbi,
 	spin_lock(&free_i->segmap_lock);
 
 	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
-		segno = find_next_zero_bit(free_i->free_segmap,
+		segno = f2fs_find_next_zero_bit(free_i->free_segmap,
 				(hint + 1) * sbi->segs_per_sec, *newseg + 1);
 		if (segno < (hint + 1) * sbi->segs_per_sec)
 			goto got_it;
 	}
 find_other_zone:
-	secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
+	secno = f2fs_find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi),
+								hint);
 	if (secno >= MAIN_SECS(sbi)) {
 		if (dir == ALLOC_RIGHT) {
-			secno = find_next_zero_bit(free_i->free_secmap,
+			secno = f2fs_find_next_zero_bit(free_i->free_secmap,
 							MAIN_SECS(sbi), 0);
 			f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
 		} else {
@@ -1103,7 +1105,7 @@ static void get_new_segment(struct f2fs_sb_info *sbi,
 			left_start--;
 			continue;
 		}
-		left_start = find_next_zero_bit(free_i->free_secmap,
+		left_start = f2fs_find_next_zero_bit(free_i->free_secmap,
 							MAIN_SECS(sbi), 0);
 		f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
 		break;
fs/f2fs/segment.h +6 -5
index d8ff069..fb58925 100644
@@ -336,7 +336,7 @@ static inline unsigned int find_next_inuse(struct free_segmap_info *free_i,
 {
 	unsigned int ret;
 	spin_lock(&free_i->segmap_lock);
-	ret = find_next_bit(free_i->free_segmap, max, segno);
+	ret = f2fs_find_next_bit(free_i->free_segmap, max, segno);
 	spin_unlock(&free_i->segmap_lock);
 	return ret;
 }
@@ -352,7 +352,7 @@ static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno)
 	clear_bit(segno, free_i->free_segmap);
 	free_i->free_segments++;
 
-	next = find_next_bit(free_i->free_segmap,
+	next = f2fs_find_next_bit(free_i->free_segmap,
 			start_segno + sbi->segs_per_sec, start_segno);
 	if (next >= start_segno + sbi->segs_per_sec) {
 		clear_bit(secno, free_i->free_secmap);
@@ -384,7 +384,7 @@ static inline void __set_test_and_free(struct f2fs_sb_info *sbi,
 	if (test_and_clear_bit(segno, free_i->free_segmap)) {
 		free_i->free_segments++;
 
-		next = find_next_bit(free_i->free_segmap,
+		next = f2fs_find_next_bit(free_i->free_segmap,
 				start_segno + sbi->segs_per_sec, start_segno);
 		if (next >= start_segno + sbi->segs_per_sec) {
 			if (test_and_clear_bit(secno, free_i->free_secmap))
@@ -607,12 +607,13 @@ static inline void check_block_count(struct f2fs_sb_info *sbi,
 	/* check bitmap with valid block count */
 	do {
 		if (is_valid) {
-			next_pos = find_next_zero_bit_le(&raw_sit->valid_map,
+			next_pos = f2fs_find_next_zero_bit_le(
+					&raw_sit->valid_map,
 					sbi->blocks_per_seg,
 					cur_pos);
 			valid_blocks += next_pos - cur_pos;
 		} else
-			next_pos = find_next_bit_le(&raw_sit->valid_map,
+			next_pos = f2fs_find_next_bit_le(&raw_sit->valid_map,
 					sbi->blocks_per_seg,
 					cur_pos);
 		cur_pos = next_pos;
Signature · 1 line
2.8.3
author maintainer
#1 lore raw

PATCH 2/3 f2fs: use __set_page_dirty_nobuffers directly

This patch replaces the copied code with original generic function.

---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -1
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION
Signature · 1 line
2.8.3
#2 lore raw

Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly

Hi Jaegeuk,

Quoted 1 line · Jaegeuk Kim

This patch replaces the copied code with original generic function.

Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.

BTW, does the original patch make memory cgroup functionality problematic?

Thanks,

Quoted 72 lines
---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -1
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION
author maintainer
#3 lore raw

Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly

Quoted 8 lines · Chao Yu

Hi Jaegeuk,

Quoted 1 line · Jaegeuk Kim

This patch replaces the copied code with original generic function.

Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.

Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.

Quoted 1 line

BTW, does the original patch make memory cgroup functionality problematic?

I don't think there is a problem, since I just copied __set_page_dirty_buffers() except page_has_buffers' stuffs.

Thank you for pointing this out. :)

Quoted 75 lines

Thanks,

Quoted 72 lines
---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -1
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION
#4 lore raw

Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly

Quoted 4 lines · Jaegeuk Kim
Quoted 3 lines · Chao Yu

Hi Jaegeuk,

On 2016/10/21 10:28, Jaegeuk Kim wrote:

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

This patch replaces the copied code with original generic function.

Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.

Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.

Why? isn't __set_page_dirty_nobuffers more fit for f2fs' non-buffer management?

Thanks,

Quoted 86 lines
Quoted 2 lines

BTW, does the original patch make memory cgroup functionality problematic?

I don't think there is a problem, since I just copied __set_page_dirty_buffers() except page_has_buffers' stuffs.

Thank you for pointing this out. :)

Quoted 67 lines

Thanks,

Quoted 64 lines
---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -0
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
Quoted 8 lines

fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210

Quoted 8 lines
fs/f2fs/f2fs.h +0 -1
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION

.

author maintainer
#5 lore raw

Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly

Quoted 7 lines · Chao Yu
Quoted 6 lines · Jaegeuk Kim
Quoted 5 lines · Chao Yu

Hi Jaegeuk,

Quoted 1 line · Jaegeuk Kim

This patch replaces the copied code with original generic function.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.

Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.

Why? isn't __set_page_dirty_nobuffers more fit for f2fs' non-buffer management?

For a while ago, when I tried to improve the performance on pmem, I could hit that __set_page_dirty_buffers() slightly improved the bandwidth comparing to __set_page_dirty_nobuffers().

When referencing the below comment written in __set_page_dirty_nobuffers(), it seems I could get that by adopting "top-down" approach instead of "bottom-up", which avoids lock contention as I guess. I couldn't do deep investigation on it though.

/*
* For address_spaces which do not use buffers. Just tag the page as dirty in
* its radix tree.
*
* This is also used when a single buffer is being dirtied: we want to set the
* page dirty in that case, but not all the buffers. This is a "bottom-up"
* dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. *
* The caller must ensure this doesn't race with truncation. Most will simply
* hold the page lock, but e.g. zap_pte_range() calls with the page mapped and
* the pte lock held, which also locks out truncation.
*/

So, I measured the performance again with fxmark on ramdisk, 8 cores, DWAL, bufferedio case. I got 2683158 works/sec w/ "top-down" over 2512609 works/sec w/ "bottom-up".

Thanks,

Quoted 89 lines

Thanks,

Quoted 86 lines
Quoted 2 lines

BTW, does the original patch make memory cgroup functionality problematic?

I don't think there is a problem, since I just copied __set_page_dirty_buffers() except page_has_buffers' stuffs.

Thank you for pointing this out. :)

Quoted 67 lines

Thanks,

Quoted 64 lines
---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -0
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
Quoted 8 lines

fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210

Quoted 8 lines
fs/f2fs/f2fs.h +0 -1
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION

.

maintainer
#6 lore raw

Re: [f2fs-dev] [PATCH 2/3] f2fs: use __set_page_dirty_nobuffers directly

Quoted 39 lines · Jaegeuk Kim
Quoted 7 lines · Chao Yu
Quoted 6 lines · Jaegeuk Kim
Quoted 5 lines · Chao Yu

Hi Jaegeuk,

Quoted 1 line · Jaegeuk Kim

This patch replaces the copied code with original generic function.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

Will we plan to do further enhance inside f2fs_set_page_dirty_nobuffers, if we don't it's better revert fe76b796fc5194cc3d57265002e3a748566d073f, as we don't need to wrap __set_page_dirty_nobuffers.

Urg. I was confused something here.
Please ignore this patch. I won't merge this patch.

Why? isn't __set_page_dirty_nobuffers more fit for f2fs' non-buffer management?

For a while ago, when I tried to improve the performance on pmem, I could hit that __set_page_dirty_buffers() slightly improved the bandwidth comparing to __set_page_dirty_nobuffers().

When referencing the below comment written in __set_page_dirty_nobuffers(), it seems I could get that by adopting "top-down" approach instead of "bottom-up", which avoids lock contention as I guess. I couldn't do deep investigation on it though.

/*
* For address_spaces which do not use buffers. Just tag the page as dirty in
* its radix tree.
*
* This is also used when a single buffer is being dirtied: we want to set the
* page dirty in that case, but not all the buffers. This is a "bottom-up"
* dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. *
* The caller must ensure this doesn't race with truncation. Most will simply
* hold the page lock, but e.g. zap_pte_range() calls with the page mapped and
* the pte lock held, which also locks out truncation.
*/

Quoted 3 lines

So, I measured the performance again with fxmark on ramdisk, 8 cores, DWAL, bufferedio case. I got 2683158 works/sec w/ "top-down" over 2512609 works/sec w/ "bottom-up".

Thanks for letting me know the history, f2fs_set_page_dirty_nobuffers and __set_page_dirty_nobuffers are almost the same except f2fs_set_page_dirty_nobuffers tries to grab mapping::private_lock additionally.

Maybe holding the private_lock does help the performance on pmem which I can't explain why it happens now...

Thanks,

Quoted 104 lines

Thanks,

Quoted 89 lines

Thanks,

Quoted 86 lines
Quoted 2 lines

BTW, does the original patch make memory cgroup functionality problematic?

I don't think there is a problem, since I just copied __set_page_dirty_buffers() except page_has_buffers' stuffs.

Thank you for pointing this out. :)

Quoted 67 lines

Thanks,

Quoted 64 lines
---
 fs/f2fs/data.c | 29 -----------------------------
 fs/f2fs/f2fs.h |  6 +++++-
 2 files changed, 5 insertions(+), 30 deletions(-)
fs/f2fs/data.c +0 -29
index 68edb47..3954315 100644
@@ -1801,35 +1801,6 @@ int f2fs_release_page(struct page *page, gfp_t wait)
 	return 1;
 }
 
-/*
- * This was copied from __set_page_dirty_buffers which gives higher performance
- * in very high speed storages. (e.g., pmem)
- */
-void f2fs_set_page_dirty_nobuffers(struct page *page)
-{
-	struct address_space *mapping = page->mapping;
-	unsigned long flags;
-
-	if (unlikely(!mapping))
-		return;
-
-	spin_lock(&mapping->private_lock);
-	lock_page_memcg(page);
-	SetPageDirty(page);
-	spin_unlock(&mapping->private_lock);
-
-	spin_lock_irqsave(&mapping->tree_lock, flags);
-	WARN_ON_ONCE(!PageUptodate(page));
-	account_page_dirtied(page, mapping);
-	radix_tree_tag_set(&mapping->page_tree,
-			page_index(page), PAGECACHE_TAG_DIRTY);
-	spin_unlock_irqrestore(&mapping->tree_lock, flags);
-	unlock_page_memcg(page);
-
-	__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
-	return;
-}
-
 static int f2fs_set_data_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
fs/f2fs/f2fs.h +5 -0
index 168f939..b66a04c 100644
@@ -1960,6 +1960,11 @@ static inline unsigned long f2fs_find_next_bit(const void *addr,
 	return find_next_bit(addr, size, offset + 2);
 }
 
+static inline void f2fs_set_page_dirty_nobuffers(struct page *page)
+{
+	__set_page_dirty_nobuffers(page);
+}
+
 #define get_inode_mode(i) \
 	((is_inode_flag_set(i, FI_ACL_MODE)) ? \
 	 (F2FS_I(i)->i_acl_mode) : ((i)->i_mode))
@@ -2200,7 +2205,6 @@ struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool);
Quoted 8 lines

fs/f2fs/f2fs.h struct page *get_new_data_page(struct inode *, struct page *, pgoff_t, bool); · line 2210

Quoted 8 lines
fs/f2fs/f2fs.h +0 -1
 int do_write_data_page(struct f2fs_io_info *);
 int f2fs_map_blocks(struct inode *, struct f2fs_map_blocks *, int, int);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *, u64, u64);
-void f2fs_set_page_dirty_nobuffers(struct page *);
 void f2fs_invalidate_page(struct page *, unsigned int, unsigned int);
 int f2fs_release_page(struct page *, gfp_t);
 #ifdef CONFIG_MIGRATION

.

------------------------------------------------------------------------------
Developer Access Program for Intel Xeon Phi Processors Access to Intel Xeon Phi processor-based developer platforms. With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
_______________________________________________
Linux-f2fs-devel mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

author maintainer
#7 lore raw

PATCH 3/3 f2fs: remove percpu_count due to performance regression

In reply to Jaegeuk Kim

This patch removes percpu_count usage due to performance regression in iozone.

Fixes 523be8a6b3 ("f2fs: use percpu_counter for page counters")
---
 fs/f2fs/debug.c | 12 ++++++------
 fs/f2fs/f2fs.h  | 12 ++++++------
 fs/f2fs/super.c | 16 +++++-----------
 3 files changed, 17 insertions(+), 23 deletions(-)
fs/f2fs/debug.c +6 -6
index 6af146c..2fdf233 100644
@@ -313,17 +313,17 @@ static int stat_show(struct seq_file *s, void *v)
 		seq_printf(s, "  - Inner Struct Count: tree: %d(%d), node: %d\n",
 				si->ext_tree, si->zombie_tree, si->ext_node);
 		seq_puts(s, "\nBalancing F2FS Async:\n");
-		seq_printf(s, "  - inmem: %4lld, wb_bios: %4d\n",
+		seq_printf(s, "  - inmem: %4d, wb_bios: %4d\n",
 			   si->inmem_pages, si->wb_bios);
-		seq_printf(s, "  - nodes: %4lld in %4d\n",
+		seq_printf(s, "  - nodes: %4d in %4d\n",
 			   si->ndirty_node, si->node_pages);
-		seq_printf(s, "  - dents: %4lld in dirs:%4d (%4d)\n",
+		seq_printf(s, "  - dents: %4d in dirs:%4d (%4d)\n",
 			   si->ndirty_dent, si->ndirty_dirs, si->ndirty_all);
-		seq_printf(s, "  - datas: %4lld in files:%4d\n",
+		seq_printf(s, "  - datas: %4d in files:%4d\n",
 			   si->ndirty_data, si->ndirty_files);
-		seq_printf(s, "  - meta: %4lld in %4d\n",
+		seq_printf(s, "  - meta: %4d in %4d\n",
 			   si->ndirty_meta, si->meta_pages);
-		seq_printf(s, "  - imeta: %4lld\n",
+		seq_printf(s, "  - imeta: %4d\n",
 			   si->ndirty_imeta);
 		seq_printf(s, "  - NATs: %9d/%9d\n  - SITs: %9d/%9d\n",
 			   si->dirty_nats, si->nats, si->dirty_sits, si->sits);
fs/f2fs/f2fs.h +6 -6
index b66a04c..27b78d4 100644
@@ -824,7 +824,7 @@ struct f2fs_sb_info {
 	atomic_t nr_wb_bios;			/* # of writeback bios */
 
 	/* # of pages, see count_type */
-	struct percpu_counter nr_pages[NR_COUNT_TYPE];
+	atomic_t nr_pages[NR_COUNT_TYPE];
 	/* # of allocated blocks */
 	struct percpu_counter alloc_valid_block_count;
 
@@ -1238,7 +1238,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 
 static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	percpu_counter_inc(&sbi->nr_pages[count_type]);
+	atomic_inc(&sbi->nr_pages[count_type]);
 
 	if (count_type == F2FS_DIRTY_DATA || count_type == F2FS_INMEM_PAGES)
 		return;
@@ -1255,7 +1255,7 @@ static inline void inode_inc_dirty_pages(struct inode *inode)
 
 static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	percpu_counter_dec(&sbi->nr_pages[count_type]);
+	atomic_dec(&sbi->nr_pages[count_type]);
 }
 
 static inline void inode_dec_dirty_pages(struct inode *inode)
@@ -1271,7 +1271,7 @@ static inline void inode_dec_dirty_pages(struct inode *inode)
 
 static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
 {
-	return percpu_counter_sum_positive(&sbi->nr_pages[count_type]);
+	return atomic_read(&sbi->nr_pages[count_type]);
 }
 
 static inline s64 get_dirty_pages(struct inode *inode)
@@ -2239,8 +2239,8 @@ struct f2fs_stat_info {
 	unsigned long long hit_largest, hit_cached, hit_rbtree;
 	unsigned long long hit_total, total_ext;
 	int ext_tree, zombie_tree, ext_node;
-	s64 ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta;
-	s64 inmem_pages;
+	int ndirty_node, ndirty_dent, ndirty_meta, ndirty_data, ndirty_imeta;
+	int inmem_pages;
 	unsigned int ndirty_dirs, ndirty_files, ndirty_all;
 	int nats, dirty_nats, sits, dirty_sits, free_nids, alloc_nids;
 	int total_count, utilization;
fs/f2fs/super.c +5 -11
index f8e2f31..9749758 100644
@@ -696,10 +696,6 @@ static void f2fs_destroy_inode(struct inode *inode)
 
 static void destroy_percpu_info(struct f2fs_sb_info *sbi)
 {
-	int i;
-
-	for (i = 0; i < NR_COUNT_TYPE; i++)
-		percpu_counter_destroy(&sbi->nr_pages[i]);
 	percpu_counter_destroy(&sbi->alloc_valid_block_count);
 	percpu_counter_destroy(&sbi->total_valid_inode_count);
 }
@@ -1449,6 +1445,7 @@ int sanity_check_ckpt(struct f2fs_sb_info *sbi)
 static void init_sb_info(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *raw_super = sbi->raw_super;
+	int i;
 
 	sbi->log_sectors_per_block =
 		le32_to_cpu(raw_super->log_sectors_per_block);
@@ -1473,6 +1470,9 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 	sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL;
 	clear_sbi_flag(sbi, SBI_NEED_FSCK);
 
+	for (i = 0; i < NR_COUNT_TYPE; i++)
+		atomic_set(&sbi->nr_pages[i], 0);
+
 	INIT_LIST_HEAD(&sbi->s_list);
 	mutex_init(&sbi->umount_mutex);
 	mutex_init(&sbi->wio_mutex[NODE]);
@@ -1488,13 +1488,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 
 static int init_percpu_info(struct f2fs_sb_info *sbi)
 {
-	int i, err;
-
-	for (i = 0; i < NR_COUNT_TYPE; i++) {
-		err = percpu_counter_init(&sbi->nr_pages[i], 0, GFP_KERNEL);
-		if (err)
-			return err;
-	}
+	int err;
 
 	err = percpu_counter_init(&sbi->alloc_valid_block_count, 0, GFP_KERNEL);
 	if (err)
Signature · 1 line
2.8.3
#8 lore raw
In reply to Jaegeuk Kim
Quoted 2 lines · Jaegeuk Kim

This patch adds checking the first two bits when searching zero or one bits to avoid costly find_next_{zero}bit operations.

Please fix this in the generic iplementations instead of adding your local workarounds.