writeback: Do not sync data dirtied after sync start
Subsystems (3)
Jan Kara
maintainerPATCH v2 1/1 writeback: Do not sync data dirtied after sync start
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between WB_SYNC_NONE and WB_SYNC_ALL pass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits in WB_SYNC_ALL phase on one fs before starting it on another fs). Because WB_SYNC_ALL pass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.
The following script reproduces the problem:
function run_writers
{
for (( i = 0; i < 10; i++ )); do
mkdir $1/dir$i
for (( j = 0; j < 40000; j++ )); do
dd if=/dev/zero of=$1/dir$i/$j bs=4k count=4 &>/dev/null
done & done
}
for dir in "$@"; do
run_writers $dir
done
sleep 40
time sync
======
Fix the problem by disregarding inodes dirtied after sync(2) was called in the WB_SYNC_ALL pass. To allow for this, sync_inodes_sb() now takes a time stamp when sync has started which is used for setting up work for flusher threads.
To give some numbers, when above script is run on two ext4 filesystems on simple SATA drive, the average sync time from 10 runs is 267.549 seconds with standard deviation 104.799426. With the patched kernel, the average sync time from 10 runs is 2.995 seconds with standard deviation 0.096.
--- fs/fs-writeback.c | 17 ++++++++--------- fs/sync.c | 15 +++++++++------ fs/xfs/xfs_super.c | 2 +- include/linux/writeback.h | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-)
fs/fs-writeback.c +8 -9
@@ -39,7 +39,7 @@struct wb_writeback_work {long nr_pages;struct super_block *sb; -unsigned long *older_than_this; +unsigned long older_than_this;enum writeback_sync_modes sync_mode;unsigned int tagged_writepages:1 ;unsigned int for_kupdate:1 ;
@@ -248,8 +248,7 @@ static int move_expired_inodes(struct list_head *delaying_queue,while (!list_empty (delaying_queue)) { inode =wb_inode (delaying_queue->prev); -if (work->older_than_this && -inode_dirtied_after (inode, *work->older_than_this)) +if (inode_dirtied_after (inode, work->older_than_this))break ;list_move (&inode->i_wb_list, &tmp); moved++;
@@ -791,12 +790,11 @@ static long wb_writeback(struct bdi_writeback *wb, {unsigned long wb_start = jiffies;long nr_pages = work->nr_pages; -unsigned long oldest_jif;struct inode *inode;long progress; - oldest_jif = jiffies; - work->older_than_this = &oldest_jif; +if (!work->older_than_this) + work->older_than_this = jiffies;spin_lock (&wb->list_lock);for (;;) {
@@ -830,10 +828,10 @@ static long wb_writeback(struct bdi_writeback *wb,* safe. */ if (work->for_kupdate) { - oldest_jif = jiffies - + work->older_than_this = jiffies -msecs_to_jiffies (dirty_expire_interval *10 ); }else if (work->for_background) - oldest_jif = jiffies; + work->older_than_this = jiffies;trace_writeback_start (wb->bdi, work);if (list_empty (&wb->b_io))
@@ -1350,13 +1348,14 @@ EXPORT_SYMBOL(try_to_writeback_inodes_sb);* This function writes and waits on any dirty inode belonging to this * super_block. */ -void sync_inodes_sb (struct super_block *sb) +void sync_inodes_sb (struct super_block *sb,unsigned long older_than_this) {DECLARE_COMPLETION_ONSTACK (done);struct wb_writeback_work work = { .sb = sb, .sync_mode =WB_SYNC_ALL , .nr_pages =LONG_MAX , + .older_than_this = older_than_this, .range_cyclic =0 , .done = &done, .reason =WB_REASON_SYNC ,
fs/sync.c +9 -6
@@ -27,10 +27,11 @@* wait == 1 case since in that case write_inode() functions do * sync_dirty_buffer() and thus effectively write one block at a time. */ -static int __sync_filesystem (struct super_block *sb,int wait) +static int __sync_filesystem (struct super_block *sb,int wait, +unsigned long start) {if (wait) -sync_inodes_sb (sb); +sync_inodes_sb (sb, start);else writeback_inodes_sb (sb,WB_REASON_SYNC );
@@ -47,6 +48,7 @@ static int __sync_filesystem(struct super_block *sb, int wait)int sync_filesystem (struct super_block *sb) {int ret; +unsigned long start = jiffies;/* * We need to be protected against the filesystem going from
@@ -60,17 +62,17 @@ int sync_filesystem(struct super_block *sb)if (sb->s_flags &MS_RDONLY )return 0 ; - ret =__sync_filesystem (sb,0 ); + ret =__sync_filesystem (sb,0 , start);if (ret <0 )return ret; -return __sync_filesystem (sb,1 ); +return __sync_filesystem (sb,1 , start); }EXPORT_SYMBOL_GPL (sync_filesystem);static void sync_inodes_one_sb (struct super_block *sb,void *arg) {if (!(sb->s_flags &MS_RDONLY )) -sync_inodes_sb (sb); +sync_inodes_sb (sb, *((unsigned long *)arg)); }static void sync_fs_one_sb (struct super_block *sb,void *arg)
@@ -102,9 +104,10 @@ static void fdatawait_one_bdev(struct block_device *bdev, void *arg)SYSCALL_DEFINE0 (sync) {int nowait =0 , wait =1 ; +unsigned long start = jiffies;wakeup_flusher_threads (0 ,WB_REASON_SYNC ); -iterate_supers (sync_inodes_one_sb,NULL ); +iterate_supers (sync_inodes_one_sb, &start);iterate_supers (sync_fs_one_sb, &nowait);iterate_supers (sync_fs_one_sb, &wait);iterate_bdevs (fdatawrite_one_bdev,NULL );
fs/xfs/xfs_super.c +1 -1
@@ -918,7 +918,7 @@ xfs_flush_inodes(struct super_block *sb = mp->m_super;if (down_read_trylock (&sb->s_umount)) { -sync_inodes_sb (sb); +sync_inodes_sb (sb, jiffies);up_read (&sb->s_umount); } }
include/linux/writeback.h +1 -1
@@ -97,7 +97,7 @@ void writeback_inodes_sb_nr(struct super_block *, unsigned long nr,int try_to_writeback_inodes_sb (struct super_block *,enum wb_reason reason);int try_to_writeback_inodes_sb_nr (struct super_block *,unsigned long nr,enum wb_reason reason); -void sync_inodes_sb (struct super_block *); +void sync_inodes_sb (struct super_block *sb,unsigned long older_than_this);void wakeup_flusher_threads (long nr_pages,enum wb_reason reason);void inode_wait_for_writeback (struct inode *inode);
Signature · 1 line
1.8.1.4
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.
Yup, that can be a problem.
Build warning form the patch:
In file included from include/trace/ftrace.h:575:0,
from include/trace/define_trace.h:90,
from include/trace/events/writeback.h:603,
from fs/fs-writeback.c:89:
include/trace/events/writeback.h: In function ¿ftrace_raw_event_writeback_queue_io¿:
include/trace/events/writeback.h:277:1: warning: initialization makes pointer from integer without a cast [enabled by default]
In file included from include/trace/ftrace.h:711:0,
from include/trace/define_trace.h:90,
from include/trace/events/writeback.h:603,
from fs/fs-writeback.c:89:include/trace/events/writeback.h: In function ¿perf_trace_writeback_queue_io¿:include/trace/events/writeback.h:277:1: warning: initialization makes pointer from integer without a cast [enabled by default]
Quoted 25 lines
The following script reproduces the problem:
function
run_writers
{
for (( i = 0; i < 10; i++ )); domkdir $1/dir$i for (( j = 0; j < 40000; j++ )); do dd if=/dev/zero of=$1/dir$i/$j bs=4k count=4 &>/dev/null done &done
}for dir in "$@"; do
run_writers$dir
donesleep 40
time sync
======Fix the problem by disregarding inodes dirtied after sync(2) was called in the
WB_SYNC_ALLpass. To allow for this,sync_inodes_sb()now takes a time stamp when sync has started which is used for setting up work for flusher threads.
Quoted 4 lines
To give some numbers, when above script is run on two ext4 filesystems on simple SATA drive, the average sync time from 10 runs is 267.549 seconds with standard deviation 104.799426. With the patched kernel, the average sync time from 10 runs is 2.995 seconds with standard deviation 0.096.
Hmmmm. 2.8 seconds on my XFS perf VM without the patch. Ok, try a smaller VM backed by single spindle of spinning rust rather than SSDs. Over 10 runs I see:
kernel min max av
vanilla 0.18s 4.46s 1.63s
patched 0.14s 0.45s 0.28s
Definitely an improvement, but nowhere near the numbers you are seeing for ext4 - maybe XFS isn't as susceptible to this problem as ext4. Nope, ext4 on an unpatched kernel gives 1.66/6.81/3.12s, (which is less than your patched kernel results :) but means so it must be something else configuration/hardware related.
Anyway, the change looks good, it just needs the above warning fixed...
Cheers,
Dave.
Signature · 6 lines
Dave Chinner [email protected] -- To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Jan Kara
maintainerQuoted 24 lines · Dave Chinner
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.Yup, that can be a problem.
Build warning form the patch:
In file included from
include/trace/ftrace.h:575:0,from include/trace/define_trace.h:90, from include/trace/events/writeback.h:603, from fs/fs-writeback.c:89: include/trace/events/writeback.h: In function ¿ftrace_raw_event_writeback_queue_io¿: include/trace/events/writeback.h:277:1: warning: initialization makes pointer from integer without a cast [enabled by default] In file included from include/trace/ftrace.h:711:0, from include/trace/define_trace.h:90, from include/trace/events/writeback.h:603, from fs/fs-writeback.c:89:
Quoted 2 lines
include/trace/events/writeback.h: In function ¿perf_trace_writeback_queue_io¿:include/trace/events/writeback.h:277:1: warning: initialization makes pointer from integer without a cast [enabled by default]
Thanks for catching this. I'll send v3 in a minute.
Quoted 38 lines
Quoted 25 lines
The following script reproduces the problem:
function
run_writers
{
for (( i = 0; i < 10; i++ )); domkdir $1/dir$i for (( j = 0; j < 40000; j++ )); do dd if=/dev/zero of=$1/dir$i/$j bs=4k count=4 &>/dev/null done &done
}for dir in "$@"; do
run_writers$dir
donesleep 40
time sync
======Fix the problem by disregarding inodes dirtied after sync(2) was called in the
WB_SYNC_ALLpass. To allow for this,sync_inodes_sb()now takes a time stamp when sync has started which is used for setting up work for flusher threads.Quoted 4 lines
To give some numbers, when above script is run on two ext4 filesystems on simple SATA drive, the average sync time from 10 runs is 267.549 seconds with standard deviation 104.799426. With the patched kernel, the average sync time from 10 runs is 2.995 seconds with standard deviation 0.096.
Hmmmm. 2.8 seconds on my XFS perf VM without the patch. Ok, try a smaller VM backed by single spindle of spinning rust rather than SSDs. Over 10 runs I see:
kernel min max av
vanilla 0.18s 4.46s 1.63s
patched 0.14s 0.45s 0.28s
Quoted 5 lines
Definitely an improvement, but nowhere near the numbers you are seeing for ext4 - maybe XFS isn't as susceptible to this problem as ext4. Nope, ext4 on an unpatched kernel gives 1.66/6.81/3.12s, (which is less than your patched kernel results :) but means so it must be something else configuration/hardware related.
Have you really used *two* (or more) busy filesystems? That makes the
problem an order of magnitude worse for me. The numbers I've posted are for such situation...
Honza
Signature · 6 lines
Jan Kara <[email protected]> SUSE Labs, CR -- To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Jan Kara
Quoted 9 lines · Dave Chinner
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.
....
Quoted 13 lines
Quoted 13 lines
Quoted 4 lines
To give some numbers, when above script is run on two ext4 filesystems on simple SATA drive, the average sync time from 10 runs is 267.549 seconds with standard deviation 104.799426. With the patched kernel, the average sync time from 10 runs is 2.995 seconds with standard deviation 0.096.
Hmmmm. 2.8 seconds on my XFS perf VM without the patch. Ok, try a smaller VM backed by single spindle of spinning rust rather than SSDs. Over 10 runs I see:
kernel min max av
vanilla 0.18s 4.46s 1.63s
patched 0.14s 0.45s 0.28s
Quoted 8 lines
Quoted 5 lines
Definitely an improvement, but nowhere near the numbers you are seeing for ext4 - maybe XFS isn't as susceptible to this problem as ext4. Nope, ext4 on an unpatched kernel gives 1.66/6.81/3.12s, (which is less than your patched kernel results :) but means so it must be something else configuration/hardware related.
Have you really used *two* (or more) busy filesystems? That makes the
problem an order of magnitude worse for me. The numbers I've posted are for such situation...
I had to bump it up to 5 active filesystems before it fell off the order of magnitude cliff. Still not the ~260s times you were seeing
- only about ~30s per sync - but you ar right in that there definitely is a load point where things go really bad. Now I've found that point, I can confirm that the patch fixes it :)
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Hi Jan,
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.
This is a very good change. An old problem that may worth noting here is that inode_dirtied_after() has a workaround for the inodes whose ->dirtied_when is never updated due to being constantly redirtied. That workaround still leaves a small time window that sync() may skip a should-be-synced inode. Since the problem existed before this patch so I'm fine with this change.
Thanks!
Jan Kara
maintainerQuoted 12 lines · Wu Fengguang
Hi Jan,
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.
Quoted 6 lines
This is a very good change. An old problem that may worth noting here is that
inode_dirtied_after()has a workaround for the inodes whose ->dirtied_when is never updated due to being constantly redirtied. That workaround still leaves a small time window thatsync()may skip a should-be-synced inode. Since the problem existed before this patch so I'm fine with this change.
Thanks for review. Do you mean the situation when jiffies ininode->dirtied_when essentially wrap around on 32-bit systems wrt current time? Yes, that is still a problem for which I don't know a better fix than the current workaround.
Honza
Signature · 2 lines
Jan Kara <[email protected]> SUSE Labs, CR
Quoted 19 lines · Jan Kara
Quoted 12 lines · Wu Fengguang
Hi Jan,
Quoted 8 lines · Jan Kara
When there are processes heavily creating small files while sync(2) is running, it can easily happen that quite some new files are created between
WB_SYNC_NONEandWB_SYNC_ALLpass of sync(2). That can happen especially if there are several busy filesystems (remember that sync traverses filesystems sequentially and waits inWB_SYNC_ALLphase on one fs before starting it on another fs). BecauseWB_SYNC_ALLpass is slow (e.g. causes a transaction commit and cache flush for each inode in ext3), resulting sync(2) times are rather large.Quoted 6 lines
This is a very good change. An old problem that may worth noting here is that
inode_dirtied_after()has a workaround for the inodes whose ->dirtied_when is never updated due to being constantly redirtied. That workaround still leaves a small time window thatsync()may skip a should-be-synced inode. Since the problem existed before this patch so I'm fine with this change.
Quoted 3 lines
Thanks for review. Do you mean the situation when jiffies in
inode->dirtied_whenessentially wrap around on 32-bit systems wrt current time?
Yes.
Quoted 2 lines
Yes, that is still a problem for which I don't know a better fix than the current workaround.
OK, that's fine. I don't have ideas on this, either.
Thanks,
Fengguang