xfs-trace-ilock-more
Can you explain the story behid this patch in SLES11SP1?
---
From: Mark Fasheh <[email protected]>
Date: Mon Oct 3 12:39:07 PDT 2011
Subject: xfs: add more ilock tracing
Patch-mainline: Never
Lets get a trace of the amount of time spent in xfs_ilock().
fs/xfs/linux-2.6/xfs_trace.h +6 -3
@@ -453,24 +453,27 @@ DEFINE_BUF_ITEM_EVENT(xfs_trans_binval);#define DEFINE_LOCK_EVENT (name) \ TRACE_EVENT(name, \ TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, \ - unsigned long caller_ip), \ - TP_ARGS(ip, lock_flags, caller_ip), \ + unsigned long start, unsigned long caller_ip), \ + TP_ARGS(ip, lock_flags, start, caller_ip), \ TP_STRUCT__entry( \ __field(dev_t, dev) \ __field(xfs_ino_t, ino) \ __field(int, lock_flags) \ + __field(unsigned long, start) \ __field(unsigned long, caller_ip) \ ), \ TP_fast_assign( \ __entry->dev = VFS_I(ip)->i_sb->s_dev; \ __entry->ino = ip->i_ino; \ __entry->lock_flags = lock_flags; \ + __entry->start = start; \ __entry->caller_ip = caller_ip; \ ), \ - TP_printk("dev %d:%d ino 0x%llx flags %s caller %pf", \ + TP_printk("dev %d:%d ino 0x%llx flags %s wait %lu caller %pf", \ MAJOR(__entry->dev), MINOR(__entry->dev), \ __entry->ino, \ __print_flags(__entry->lock_flags, "|", XFS_LOCK_FLAGS), \ + (jiffies - __entry->start), \ (void *)__entry->caller_ip) \ )
fs/xfs/xfs_iget.c +13 -4
@@ -590,6 +590,8 @@ xfs_ilock( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -611,7 +613,7 @@ xfs_ilock(else if (lock_flags &XFS_ILOCK_SHARED )mraccess_nested (&ip->i_lock,XFS_ILOCK_DEP (lock_flags)); -trace_xfs_ilock (ip, lock_flags, _RET_IP_); +trace_xfs_ilock (ip, lock_flags, start, _RET_IP_); }/*
@@ -631,6 +633,8 @@ xfs_ilock_nowait( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -656,7 +660,7 @@ xfs_ilock_nowait(if (!mrtryaccess (&ip->i_lock)) goto out_undo_iolock; } -trace_xfs_ilock_nowait (ip, lock_flags, _RET_IP_); +trace_xfs_ilock_nowait (ip, lock_flags, start, _RET_IP_);return 1 ; out_undo_iolock:
@@ -684,7 +688,10 @@ voidxfs_iunlock ( xfs_inode_t *ip, uint lock_flags) + { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -718,7 +725,7 @@ xfs_iunlock(xfs_trans_unlocked_item (ip->i_itemp->ili_item.li_ailp, (xfs_log_item_t *)(ip->i_itemp)); } -trace_xfs_iunlock (ip, lock_flags, _RET_IP_); +trace_xfs_iunlock (ip, lock_flags, start, _RET_IP_); }/*
@@ -730,6 +737,8 @@ xfs_ilock_demote( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +ASSERT (lock_flags & (XFS_IOLOCK_EXCL |XFS_ILOCK_EXCL ));ASSERT ((lock_flags & ~(XFS_IOLOCK_EXCL |XFS_ILOCK_EXCL )) ==0 );
@@ -738,7 +747,7 @@ xfs_ilock_demote(if (lock_flags &XFS_IOLOCK_EXCL )mrdemote (&ip->i_iolock); -trace_xfs_ilock_demote (ip, lock_flags, _RET_IP_); +trace_xfs_ilock_demote (ip, lock_flags, start, _RET_IP_); }#ifdef DEBUG
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Hey Christoph,
Quoted 1 line · Christoph Hellwig
Can you explain the story behid this patch in SLES11SP1?
We were looking at some performance issues and needed a bit more information on the amount of time spent in ilock. I can give you more specifics if you want, I just have to dig up the e-mails (it's been a while).
Regards, Mark
--
Mark Fasheh
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Quoted 9 lines · Mark Fasheh
Hey Christoph,
Quoted 1 line · Christoph Hellwig
Can you explain the story behid this patch in SLES11SP1?
We were looking at some performance issues and needed a bit more information on the amount of time spent in ilock. I can give you more specifics if you want, I just have to dig up the e-mails (it's been a while).
That's pretty much the explanation. With heavy reader load, buffered writes were stalling for 80 ms and sometimes longer. I suspected it was contention on the ilock and the tracing with that patch demonstrated a delay there. Since we were chasing a similar issue at another site, it seemed worthwhile to just keep it around. We're still tracking down the cause. I'm not sure if more recent kernels have the same issue as there's been quite a lot of churn.
- -Jeff
- --
Jeff Mahoney
SUSE Labs
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.18 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/
iQIcBAEBAgAGBQJO6PfwAAoJEB57S2MheeWyE+MQAIeh/VUfWExpjsgdBbkSj4jv 7sJQW2WweYpHHZdLnGx5Pn4cDO6/2YyqNXfwAGZk2yoXTcycykR7ZVHLGEyyNDLV BrKUvcDG/5XYTx3B64yPge8rNY9YIiy2e/c+pbwPeSnDPMY9976igqbrC5qaJ31a 1a/REm/MpDkGV7NcS4XYjbZwl0bP6BBJib0fnCrvkmWo7mtuzwz1XMpORGLWltOg EhRTTgh+HRc1zlqknz5+ceMghky+GTp56/3YPslbco61uI9XmKzLc/bi1iNgda24 gy6wAIRk0EwPd9o2Wl+Jvvm79YvTE2SvkjDzHz3exOAX3JTMVnF//7bjVfPg7A9f xyHQ7GQJRWQwIXuGVqi7JciN5/5IKOdkrGILIIC1osFjr7z894xGgQejNhqxuXNE B06WUJ2PLYEOCKLCnH75OwCsDRqdj7N+yPbxGJIZeAjdiH6r2tanAAtYAtqdn+3T J7sdsm/Kvod4wgKBLJWk/rvTI/i1pjoqNaeF0rdXsQbqGI7C9OKeNfIjilt0HT8h 9EIaoy3KjnGmAU1pouHZro2hR1Ec8Ni09DM5GMQeGTIAPvgTgP8oHB/MPFIM9Jnl QxfLJ5o3bxvB3u2aDJQta3+V/9TQPfELtokjjk34GjAcaxBZT7mTy8lVOHVL1Srg 86cHVs6PDRXQf7CeoX+N
=Cjlo
-----END PGP SIGNATURE-----
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 14 lines · Jeff Mahoney
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1Quoted 9 lines · Mark Fasheh
Hey Christoph,
Quoted 1 line · Christoph Hellwig
Can you explain the story behid this patch in SLES11SP1?
We were looking at some performance issues and needed a bit more information on the amount of time spent in ilock. I can give you more specifics if you want, I just have to dig up the e-mails (it's been a while).
Quoted 7 lines
That's pretty much the explanation. With heavy reader load, buffered writes were stalling for 80 ms and sometimes longer. I suspected it was contention on the ilock and the tracing with that patch demonstrated a delay there. Since we were chasing a similar issue at another site, it seemed worthwhile to just keep it around. We're still tracking down the cause. I'm not sure if more recent kernels have the same issue as there's been quite a lot of churn.
I'm not surprised - there's nothing really guaranteeing bound shared vs exclusive access to the ilock. It's all down to the read/write bias of the rwsem - readers will hold off the writer for some time. Still, it would be nice to see a trace from such a holdoff to confirm this is actually the case...
FWIW, if you have an app that requires concurrent, low latency reads and writes to the same file, that's what the XFS Direct IO was designed for - in most cases the iolock is taken in shared mode for both read and write, and so such hold-offs don't generally happen...
Cheers,
Dave.
Signature · 7 lines
Dave Chinner [email protected] _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Quoted 24 lines · Dave Chinner
Quoted 14 lines · Jeff Mahoney
-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1
Quoted 10 lines · Mark Fasheh
Hey Christoph,
Quoted 1 line · Christoph Hellwig
Can you explain the story behid this patch in SLES11SP1?
We were looking at some performance issues and needed a bit more information on the amount of time spent in ilock. I can give you more specifics if you want, I just have to dig up the e-mails (it's been a while).
Quoted 8 lines
That's pretty much the explanation. With heavy reader load, buffered writes were stalling for 80 ms and sometimes longer. I suspected it was contention on the ilock and the tracing with that patch demonstrated a delay there. Since we were chasing a similar issue at another site, it seemed worthwhile to just keep it around. We're still tracking down the cause. I'm not sure if more recent kernels have the same issue as there's been quite a lot of churn.
Quoted 5 lines
I'm not surprised - there's nothing really guaranteeing bound shared vs exclusive access to the ilock. It's all down to the read/write bias of the rwsem - readers will hold off the writer for some time. Still, it would be nice to see a trace from such a holdoff to confirm this is actually the case...
Sure, let me dig it up. We do actually have real results from this workload with the trace running.
Quoted 5 lines
FWIW, if you have an app that requires concurrent, low latency reads and writes to the same file, that's what the XFS Direct IO was designed for - in most cases the iolock is taken in shared mode for both read and write, and so such hold-offs don't generally happen...
You don't need to convince me. I agree with you. :)
- -Jeff
- --
Jeff Mahoney
SUSE Labs
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.18 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/
iQIcBAEBAgAGBQJO6SZeAAoJEB57S2MheeWyzfgP/3gbX++mlm3pY8iRlWkqEcy4 77OBJyMn0jxUqQkLw3nmrCcIMVJxKREJhtBoNl93mBdN8HZvx1X7nTUst769Ksf/ NsjQd3NMyv/pymjuJBB81APnyc2diM/fz/t5f1gh4xHljKG9rUJS8ogRQ65QBbMG kDbwtgaXYMZpUpf/GAF7Q0jxefNZbs82BfSsGqDH6e+fdymqD9/GWewyIFCl3tVR yOk80v4izDwlgtwtm/anq18nqePZl8M3ktwtucpAR/cxZ79nOryGxuqaD5r7GOam bo/29A5x0UKptsxHKnEvf6JIlznVWR6RzheyV/A6d3gPFkMa8550NBFMmWObjh7h DA/fcsZFhzD1LZI9BXU58ib6nT3kmtQMHmdsuwPsX5XPLzdymMNFD/yUDP7vhMWC OR1kkLPBSGeCFK7m0IeScO+jCnVsadhzkeS3rMjbUYCSw3+4OGf21sAJnfsMasPb Z2ZbbSZdUGItb8bK1lx7WEwST8KC5UYe2n3LIoH8CJ8qH+FrzX0pQKx+Gm2SVoMK nZZKDbtwN9+ZWRs2p3tEuT34sb/XGFV1XnEBPAv272XAjZKvjwBD8JZF92fQxl8a e2yjy7+EsqmZXNguTcs+KhLJ69QkKkg6J0rozPgUNHmA+KGhSDCgRI2kMCPDL3MO uKZ2umow6YYTofS6mBp8
=zmC1
-----END PGP SIGNATURE-----
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 7 lines · Jeff Mahoney
That's pretty much the explanation. With heavy reader load, buffered writes were stalling for 80 ms and sometimes longer. I suspected it was contention on the ilock and the tracing with that patch demonstrated a delay there. Since we were chasing a similar issue at another site, it seemed worthwhile to just keep it around. We're still tracking down the cause. I'm not sure if more recent kernels have the same issue as there's been quite a lot of churn.
Ok. I was a bit surprised that it was one out of only two XFS updates that went into a recent SLES security errata, and I never heard about it before.
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 8 lines · Mark Fasheh
Hey Christoph,
Quoted 1 line · Christoph Hellwig
Can you explain the story behid this patch in SLES11SP1?
We were looking at some performance issues and needed a bit more information on the amount of time spent in ilock. I can give you more specifics if you want, I just have to dig up the e-mails (it's been a while).
Given that the patch is fairly clean can you send it to us for inclusion?
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 2 lines · Christoph Hellwig
Given that the patch is fairly clean can you send it to us for inclusion?
Here goes. I updated the patch for 3.2 and of course gave it a quick test.
--Mark
--
Mark Fasheh
From: Mark Fasheh <[email protected]>
[PATCH] xfs: add more ilock tracing
Let's get a trace of the amount of time spent in xfs_ilock(). This has helped us (SUSE) in investigating read/write performance issues in the past when ilock() contention has come up as a possibile issue.
--- fs/xfs/xfs_iget.c | 16 ++++++++++++---- fs/xfs/xfs_trace.h | 13 ++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-)
fs/xfs/xfs_iget.c +12 -4
@@ -543,6 +543,8 @@ xfs_ilock( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -564,7 +566,7 @@ xfs_ilock(else if (lock_flags &XFS_ILOCK_SHARED )mraccess_nested (&ip->i_lock,XFS_ILOCK_DEP (lock_flags)); -trace_xfs_ilock (ip, lock_flags, _RET_IP_); +trace_xfs_ilock (ip, lock_flags, start, _RET_IP_); }/*
@@ -584,6 +586,8 @@ xfs_ilock_nowait( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -609,7 +613,7 @@ xfs_ilock_nowait(if (!mrtryaccess (&ip->i_lock)) goto out_undo_iolock; } -trace_xfs_ilock_nowait (ip, lock_flags, _RET_IP_); +trace_xfs_ilock_nowait (ip, lock_flags, start, _RET_IP_);return 1 ; out_undo_iolock:
@@ -638,6 +642,8 @@ xfs_iunlock( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +/* * You can't set both SHARED and EXCL for the same lock, * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_ILOCK_SHARED,
@@ -671,7 +677,7 @@ xfs_iunlock(xfs_trans_unlocked_item (ip->i_itemp->ili_item.li_ailp, (xfs_log_item_t *)(ip->i_itemp)); } -trace_xfs_iunlock (ip, lock_flags, _RET_IP_); +trace_xfs_iunlock (ip, lock_flags, start, _RET_IP_); }/*
@@ -683,6 +689,8 @@ xfs_ilock_demote( xfs_inode_t *ip, uint lock_flags) { +unsigned long start = jiffies; +ASSERT (lock_flags & (XFS_IOLOCK_EXCL |XFS_ILOCK_EXCL ));ASSERT ((lock_flags & ~(XFS_IOLOCK_EXCL |XFS_ILOCK_EXCL )) ==0 );
@@ -691,7 +699,7 @@ xfs_ilock_demote(if (lock_flags &XFS_IOLOCK_EXCL )mrdemote (&ip->i_iolock); -trace_xfs_ilock_demote (ip, lock_flags, _RET_IP_); +trace_xfs_ilock_demote (ip, lock_flags, start, _RET_IP_); }#ifdef DEBUG
fs/xfs/xfs_trace.h +8 -5
@@ -507,32 +507,35 @@ DEFINE_BUF_ITEM_EVENT(xfs_trans_binval);DECLARE_EVENT_CLASS (xfs_lock_class, TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, - unsigned long caller_ip), -TP_ARGS (ip, lock_flags, caller_ip), + unsigned long start, unsigned long caller_ip), +TP_ARGS (ip,lock_flags, start, caller_ip),TP_STRUCT__entry ( __field(dev_t, dev) __field(xfs_ino_t, ino) __field(int, lock_flags) + __field(unsigned long, start) __field(unsigned long, caller_ip) ),TP_fast_assign ( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->ino = ip->i_ino; __entry->lock_flags = lock_flags; + __entry->start = start; __entry->caller_ip = caller_ip; ), -TP_printk ("dev %d:%d ino 0x%llx flags %s caller %pf" , +TP_printk ("dev %d:%d ino 0x%llx flags %s wait %lu caller %pf" ,MAJOR (__entry->dev),MINOR (__entry->dev), __entry->ino,__print_flags (__entry->lock_flags,"|" ,XFS_LOCK_FLAGS ), + (jiffies - __entry->start), (void *)__entry->caller_ip) )#define DEFINE_LOCK_EVENT (name) \ DEFINE_EVENT(xfs_lock_class, name, \ TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, \ - unsigned long caller_ip), \ - TP_ARGS(ip, lock_flags, caller_ip)) + unsigned long start, unsigned long caller_ip), \ + TP_ARGS(ip, lock_flags, start, caller_ip))DEFINE_LOCK_EVENT (xfs_ilock);DEFINE_LOCK_EVENT (xfs_ilock_nowait);DEFINE_LOCK_EVENT (xfs_ilock_demote);
Signature · 6 lines
1.7.6 _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 19 lines · Mark Fasheh
Quoted 2 lines · Christoph Hellwig
Given that the patch is fairly clean can you send it to us for inclusion?
Here goes. I updated the patch for 3.2 and of course gave it a quick test.
--Mark--
Mark FashehFrom: Mark Fasheh <[email protected]>
[PATCH] xfs: add more ilock tracing
Let's get a trace of the amount of time spent in
xfs_ilock(). This has helped us (SUSE) in investigating read/write performance issues in the past whenilock()contention has come up as a possibile issue.Signed-off-by Mark Fasheh <[email protected]>
....
Quoted 20 lines
fs/xfs/xfs_trace.h +3
-2@@ -507,32 +507,35 @@ DEFINE_BUF_ITEM_EVENT(xfs_trans_binval); DECLARE_EVENT_CLASS(xfs_lock_class, TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, - unsigned long caller_ip), - TP_ARGS(ip, lock_flags, caller_ip), + unsigned long start, unsigned long caller_ip), + TP_ARGS(ip,lock_flags, start, caller_ip), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_ino_t, ino) __field(int, lock_flags) + __field(unsigned long, start) __field(unsigned long, caller_ip) ), TP_fast_assign( __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->ino = ip->i_ino;
fs/xfs/xfs_trace.h DEFINE_BUF_ITEM_EVENT(xfs_trans_binval); · line 530
Quoted 10 lines
fs/xfs/xfs_trace.h +3
-1__entry->lock_flags = lock_flags; + __entry->start = start; __entry->caller_ip = caller_ip; ), - TP_printk("dev %d:%d ino 0x%llx flags %s caller %pf", + TP_printk("dev %d:%d ino 0x%llx flags %s wait %lu caller %pf", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __print_flags(__entry->lock_flags, "|", XFS_LOCK_FLAGS), + (jiffies - __entry->start),
I'm not sure this is valid - the printk format can be used by tools outside the kernel at a later time (e.g. they read from the kernel in binary format). If you want this to be done, the it needs to be calculated in the TP_fast_assign() macro.
Indeed, you are printing is the time delta between the start of the operation and the completion. in that case, the delta should be what is recorded in the trace point, not calculated implicitly inside the format string of the trace point. i.e callers do:
trace_xfs_iunlock(ip, lock_flags, jiffies - start, _RET_IP_);
to pass an explicit delta value to the trace point...
Cheers,
Dave.
Signature · 7 lines
Dave Chinner [email protected] _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs