iversion: Use atomic64_try_cmpxchg
PATCH 1/1 iversion: Use atomic64_try_cmpxchg
Use atomic64_try_cmpxchg instead ofatomic64_cmpxchg (*ptr, old, new) == old in inode_set_max_iversion_raw, inode_maybe_inc_version and inode_query_iversion. x86 CMPXCHG instruction returns success in ZF flag, so this change saves a compare after cmpxchg (and related move instruction in front of cmpxchg).
Also, try_cmpxchg implicitly assigns old *ptr value to "old" when cmpxchg fails, enabling further code simplifications.
The loop in inode_maybe_inc_iversion improves from:
5563: 48 89 ca mov %rcx,%rdx
5566: 48 89 c8 mov %rcx,%rax
5569: 48 83 e2 fe and $0xfffffffffffffffe,%rdx
556d: 48 83 c2 02 add $0x2,%rdx
5571: f0 48 0f b1 16 lock cmpxchg %rdx,(%rsi)
5576: 48 39 c1 cmp %rax,%rcx
5579: 0f 84 85 fc ff ff je 5204 <...>
557f: 48 89 c1 mov %rax,%rcx
5582: eb df jmp 5563 <...>to:
5563: 48 89 c2 mov %rax,%rdx
5566: 48 83 e2 fe and $0xfffffffffffffffe,%rdx
556a: 48 83 c2 02 add $0x2,%rdx
556e: f0 48 0f b1 11 lock cmpxchg %rdx,(%rcx)
5573: 0f 84 8b fc ff ff je 5204 <...>
5579: eb e8 jmp 5563 <...>Signed-off-by Uros Bizjak <[email protected]>
Cc Andrew Morton <[email protected]>
--- include/linux/iversion.h | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-)
include/linux/iversion.h +9 -23
@@ -123,17 +123,12 @@ inode_peek_iversion_raw(const struct inode *inode)static inline void inode_set_max_iversion_raw (struct inode *inode,u64 val) { -u64 cur, old; +u64 cur =inode_peek_iversion_raw (inode); - cur =inode_peek_iversion_raw (inode); -for (;;) { +do {if (cur > val)break ; - old =atomic64_cmpxchg (&inode->i_version, cur, val); -if (likely (old == cur)) -break ; - cur = old; - } + }while (!atomic64_try_cmpxchg (&inode->i_version, &cur, val)); }/**
@@ -197,7 +192,7 @@ inode_set_iversion_queried(struct inode *inode, u64 val)static inline bool inode_maybe_inc_iversion (struct inode *inode,bool force) { -u64 cur, old, new; +u64 cur, new;/* * The i_version field is not strictly ordered with any other inode
@@ -211,19 +206,14 @@ inode_maybe_inc_iversion(struct inode *inode, bool force)*/ smp_mb (); cur =inode_peek_iversion_raw (inode); -for (;;) { +do {/* If flag is clear then we needn't do anything */ if (!force && !(cur &I_VERSION_QUERIED ))return false;/* Since lowest bit is flag, add 2 to avoid it */ new = (cur & ~I_VERSION_QUERIED ) +I_VERSION_INCREMENT ; - - old =atomic64_cmpxchg (&inode->i_version, cur, new); -if (likely (old == cur)) -break ; - cur = old; - } + }while (!atomic64_try_cmpxchg (&inode->i_version, &cur, new));return true; }
@@ -304,10 +294,10 @@ inode_peek_iversion(const struct inode *inode)static inline u64 inode_query_iversion (struct inode *inode) { -u64 cur, old, new; +u64 cur, new; cur =inode_peek_iversion_raw (inode); -for (;;) { + do { /* If flag is already set, then no need to swap */ if (cur & I_VERSION_QUERIED) { /*
@@ -320,11 +310,7 @@ inode_query_iversion(struct inode *inode) } new = cur |I_VERSION_QUERIED ; - old =atomic64_cmpxchg (&inode->i_version, cur, new); -if (likely (old == cur)) -break ; - cur = old; - } + }while (!atomic64_try_cmpxchg (&inode->i_version, &cur, new));return cur >>I_VERSION_QUERIED_SHIFT ; }
Signature · 1 line
2.37.1