fs: break out inode LRU operations from inode_lock V2
Subsystems (3)
Hi Al,
The following patches break the inode LRU operations and the first half of iput_final() out from under the inode_lock. The patches should apply to your current merge-stem tree.
Version 2:
- make i_state checks/__iget() atomic under inode->i_lock
- merge dispose_one_inode() into evict()
- move inode hash removal after ->evict_inode call.
- always take the inode_lru_lock() when checking whether the inode is on the lru or not.
PATCH 1/4 fs: protect inode->i_state with inode->i_lock
From: Dave Chinner <[email protected]>
Protect inode state transitions and validity checks with the inode->i_lock. This enables us to make inode state transitions independently of the inode_lock and is the first step to peeling away the inode_lock from the code.
This requires that __iget() is done atomically with i_state checks during list traversals so that we don't race with another thread marking the inode I_FREEING between the state check and grabbing the reference.
--- fs/drop_caches.c | 9 +++-- fs/fs-writeback.c | 42 +++++++++++++++++----- fs/inode.c | 93 +++++++++++++++++++++++++++++++++++++++-------- fs/notify/inode_mark.c | 21 ++++++++--- fs/quota/dquot.c | 13 ++++--- 5 files changed, 138 insertions(+), 40 deletions(-)
fs/drop_caches.c +6 -3
@@ -18,11 +18,14 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)spin_lock (&inode_lock);list_for_each_entry (inode, &sb->s_inodes, i_sb_list) { -if (inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) -continue ; -if (inode->i_mapping->nrpages ==0 ) +spin_lock (&inode->i_lock); +if ((inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) || + (inode->i_mapping->nrpages ==0 )) { +spin_unlock (&inode->i_lock);continue ; + }__iget (inode); +spin_unlock (&inode->i_lock);spin_unlock (&inode_lock);invalidate_mapping_pages (inode->i_mapping,0 ,-1 );iput (toput_inode);
fs/fs-writeback.c +33 -9
149 lines not on this page · raw message
fs/inode.c +77 -16
302 lines not on this page · raw message
fs/notify/inode_mark.c +15 -6
@@ -249,8 +249,11 @@ void fsnotify_unmount_inodes(struct list_head *list)* I_WILL_FREE, or I_NEW which is fine because by that point * the inode cannot have any associated watches. */ -if (inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) +spin_lock (&inode->i_lock); +if (inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) { +spin_unlock (&inode->i_lock);continue ; + }/* * If i_count is zero, the inode cannot have any watches and
@@ -258,8 +261,10 @@ void fsnotify_unmount_inodes(struct list_head *list)* evict all inodes with zero i_count from icache which is * unnecessarily violent and may in fact be illegal to do. */ -if (!atomic_read (&inode->i_count)) +if (!atomic_read (&inode->i_count)) { +spin_unlock (&inode->i_lock);continue ; + } need_iput_tmp = need_iput; need_iput =NULL ;
@@ -269,13 +274,17 @@ void fsnotify_unmount_inodes(struct list_head *list)__iget (inode); else need_iput_tmp =NULL ; +spin_unlock (&inode->i_lock);/* In case the dropping of a reference would nuke next_i. */ if ((&next_i->i_sb_list != list) && -atomic_read (&next_i->i_count) && - !(next_i->i_state & (I_FREEING |I_WILL_FREE ))) { -__iget (next_i); - need_iput = next_i; +atomic_read (&next_i->i_count)) { +spin_lock (&next_i->i_lock); +if (!(next_i->i_state & (I_FREEING |I_WILL_FREE ))) { +__iget (next_i); + need_iput = next_i; + } +spin_unlock (&next_i->i_lock); }/*
fs/quota/dquot.c +7 -6
@@ -898,18 +898,19 @@ static void add_dquot_ref(struct super_block *sb, int type)spin_lock (&inode_lock);list_for_each_entry (inode, &sb->s_inodes, i_sb_list) { -if (inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) +spin_lock (&inode->i_lock); +if ((inode->i_state & (I_FREEING |I_WILL_FREE |I_NEW )) || + !atomic_read (&inode->i_writecount) || + !dqinit_needed (inode, type)) { +spin_unlock (&inode->i_lock);continue ; + }#ifdef CONFIG_QUOTA_DEBUG if (unlikely (inode_get_rsv_space (inode) >0 )) reserved =1 ;#endif -if (!atomic_read (&inode->i_writecount)) -continue ; -if (!dqinit_needed (inode, type)) -continue ; -__iget (inode); +spin_unlock (&inode->i_lock);spin_unlock (&inode_lock);iput (old_inode);
Signature · 1 line
1.7.1
PATCH 2/4 fs: factor inode disposal
In reply to Dave ChinnerFrom: Dave Chinner <[email protected]>
We have a couple of places that dispose of inodes. factor the disposal into evict() to isolate this code and make it simpler to peel away the inode_lock from the code.
While doing this, change the logic flow in iput_final() to separate the different cases that need to be handled to make the transitions the inode goes through more obvious.
--- fs/inode.c | 89 +++++++++++++++++++++++++---------------------------------- 1 files changed, 38 insertions(+), 51 deletions(-)
fs/inode.c +38 -51
@@ -408,17 +408,6 @@ void __insert_inode_hash(struct inode *inode, unsigned long hashval)EXPORT_SYMBOL (__insert_inode_hash);/** -* __remove_inode_hash - remove an inode from the hash -* @inode: inode to unhash -* -* Remove an inode from the superblock. -*/ -static void __remove_inode_hash (struct inode *inode) -{ -hlist_del_init (&inode->i_hash); -} - -/** * remove_inode_hash - remove an inode from the hash * @inode: inode to unhash *
@@ -446,10 +435,26 @@ void end_writeback(struct inode *inode) }EXPORT_SYMBOL (end_writeback); +/* +* Free the inode passed in, removing it from the lists it is still connected +* to. We remove any pages still attached to the inode and wait for any IO that +* is still in progress before finally destroying the inode. +* +* An inode must already be marked I_FREEING so that we avoid the inode being +* moved back onto lists if we race with other code that manipulates the lists +* (e.g. writeback_single_inode). The caller is responsible for setting this. +*/ static void evict (struct inode *inode) {const struct super_operations *op = inode->i_sb->s_op; +BUG_ON (!(inode->i_state &I_FREEING )); + +spin_lock (&inode_lock); +list_del_init (&inode->i_wb_list); +__inode_sb_list_del (inode); +spin_unlock (&inode_lock); +if (op->evict_inode) { op->evict_inode (inode); }else {
@@ -461,6 +466,11 @@ static void evict(struct inode *inode)bd_forget (inode);if (S_ISCHR (inode->i_mode) && inode->i_cdev)cd_forget (inode); + +remove_inode_hash (inode); +wake_up_inode (inode); +BUG_ON (inode->i_state != (I_FREEING |I_CLEAR )); +destroy_inode (inode); }/*
@@ -479,14 +489,6 @@ static void dispose_list(struct list_head *head)list_del_init (&inode->i_lru);evict (inode); - -spin_lock (&inode_lock); -__remove_inode_hash (inode); -__inode_sb_list_del (inode); -spin_unlock (&inode_lock); - -wake_up_inode (inode); -destroy_inode (inode); } }
@@ -524,11 +526,10 @@ void evict_inodes(struct super_block *sb)spin_unlock (&inode->i_lock);/* -* Move the inode off the IO lists and LRU once I_FREEING is -* set so that it won't get moved back on there if it is dirty. +* Move the inode off the LRU once I_FREEING is set so that it +* won't get moved back on. */ list_move (&inode->i_lru, &dispose); -list_del_init (&inode->i_wb_list); }spin_unlock (&inode_lock);
@@ -570,11 +571,10 @@ int invalidate_inodes(struct super_block *sb)spin_unlock (&inode->i_lock);/* -* Move the inode off the IO lists and LRU once I_FREEING is -* set so that it won't get moved back on there if it is dirty. +* Move the inode off the LRU once I_FREEING is +* set so that it won't get moved back on. */ list_move (&inode->i_lru, &dispose); -list_del_init (&inode->i_wb_list); }spin_unlock (&inode_lock);
@@ -673,11 +673,10 @@ static void prune_icache(int nr_to_scan)spin_unlock (&inode->i_lock);/* -* Move the inode off the IO lists and LRU once I_FREEING is -* set so that it won't get moved back on there if it is dirty. +* Move the inode off the LRU once I_FREEING is +* set so that it won't get moved back on. */ list_move (&inode->i_lru, &freeable); -list_del_init (&inode->i_wb_list);percpu_counter_dec (&nr_inodes_unused); }if (current_is_kswapd ())
@@ -1409,16 +1408,16 @@ static void iput_final(struct inode *inode)else drop =generic_drop_inode (inode); +if (!drop && (sb->s_flags &MS_ACTIVE )) { + inode->i_state |=I_REFERENCED ; +if (!(inode->i_state & (I_DIRTY |I_SYNC ))) +inode_lru_list_add (inode); +spin_unlock (&inode->i_lock); +spin_unlock (&inode_lock); +return ; + } +if (!drop) { -if (sb->s_flags &MS_ACTIVE ) { - inode->i_state |=I_REFERENCED ; -if (!(inode->i_state & (I_DIRTY |I_SYNC ))) { -inode_lru_list_add (inode); - } -spin_unlock (&inode->i_lock); -spin_unlock (&inode_lock); -return ; - } inode->i_state |=I_WILL_FREE ;spin_unlock (&inode->i_lock);spin_unlock (&inode_lock);
@@ -1427,26 +1426,14 @@ static void iput_final(struct inode *inode)spin_lock (&inode->i_lock);WARN_ON (inode->i_state &I_NEW ); inode->i_state &= ~I_WILL_FREE ; -__remove_inode_hash (inode); } inode->i_state |=I_FREEING ; -spin_unlock (&inode->i_lock); - -/* -* Move the inode off the IO lists and LRU once I_FREEING is -* set so that it won't get moved back on there if it is dirty. -*/ inode_lru_list_del (inode); -list_del_init (&inode->i_wb_list); - -__inode_sb_list_del (inode); +spin_unlock (&inode->i_lock);spin_unlock (&inode_lock); +evict (inode); -remove_inode_hash (inode); -wake_up_inode (inode); -BUG_ON (inode->i_state != (I_FREEING |I_CLEAR )); -destroy_inode (inode); }/**
Signature · 1 line
1.7.1
PATCH 3/4 fs: Lock the inode LRU list separately
In reply to Dave ChinnerFrom: Dave Chinner <[email protected]>
Introduce the inode_lru_lock to protect the inode_lru list. This lock is nested inside the inode->i_lock to allow the inode to be added to the LRU list in iput_final without needing to deal with lock inversions. This keeps iput_final() clean and neat.
Further, where marking the inode I_FREEING and removing it from the LRU, move the LRU list manipulation within the inode->i_lock to keep the list manipulation consistent with iput_final. This also means that most of the open coded LRU list removal + unused inode accounting can now use the inode_lru_list_del() wrappers which cleans the code up further.
However, this locking change means what the LRU traversal in prune_icache() inverts this lock ordering and needs to use trylock semantics on the inode->i_lock to avoid deadlocking. In these cases, if we fail to lock the inode we move it to the back of the LRU to prevent spinning on it.
--- fs/inode.c | 45 +++++++++++++++++++++++++++++---------------- 1 files changed, 29 insertions(+), 16 deletions(-)
fs/inode.c +29 -16
@@ -30,10 +30,13 @@* * inode->i_lock protects: * inode->i_state, __iget() +* inode_lru_lock protects: +* inode_lru, inode->i_lru * * Lock ordering: * inode_lock * inode->i_lock +* inode_lru_lock */ /*
@@ -83,6 +86,7 @@ static unsigned int i_hash_shift __read_mostly;*/ static LIST_HEAD (inode_lru); +static DEFINE_SPINLOCK (inode_lru_lock);static struct hlist_head *inode_hashtable__read_mostly ;/*
@@ -343,18 +347,22 @@ EXPORT_SYMBOL(ihold);static void inode_lru_list_add (struct inode *inode) { +spin_lock (&inode_lru_lock);if (list_empty (&inode->i_lru)) {list_add (&inode->i_lru, &inode_lru);percpu_counter_inc (&nr_inodes_unused); } +spin_unlock (&inode_lru_lock); }static void inode_lru_list_del (struct inode *inode) { +spin_lock (&inode_lru_lock);if (!list_empty (&inode->i_lru)) {list_del_init (&inode->i_lru);percpu_counter_dec (&nr_inodes_unused); } +spin_unlock (&inode_lru_lock); }static inline void __inode_sb_list_add (struct inode *inode)
@@ -521,15 +529,10 @@ void evict_inodes(struct super_block *sb) } inode->i_state |=I_FREEING ; -if (!(inode->i_state & (I_DIRTY |I_SYNC ))) -percpu_counter_dec (&nr_inodes_unused); +inode_lru_list_del (inode);spin_unlock (&inode->i_lock); -/* -* Move the inode off the LRU once I_FREEING is set so that it -* won't get moved back on. -*/ -list_move (&inode->i_lru, &dispose); +list_add (&inode->i_lru, &dispose); }spin_unlock (&inode_lock);
@@ -566,15 +569,10 @@ int invalidate_inodes(struct super_block *sb) } inode->i_state |=I_FREEING ; -if (!(inode->i_state & (I_DIRTY |I_SYNC ))) -percpu_counter_dec (&nr_inodes_unused); +inode_lru_list_del (inode);spin_unlock (&inode->i_lock); -/* -* Move the inode off the LRU once I_FREEING is -* set so that it won't get moved back on. -*/ -list_move (&inode->i_lru, &dispose); +list_add (&inode->i_lru, &dispose); }spin_unlock (&inode_lock);
@@ -621,6 +619,7 @@ static void prune_icache(int nr_to_scan)down_read (&iprune_sem);spin_lock (&inode_lock); +spin_lock (&inode_lru_lock);for (nr_scanned =0 ; nr_scanned < nr_to_scan; nr_scanned++) {struct inode *inode;
@@ -630,10 +629,19 @@ static void prune_icache(int nr_to_scan) inode = list_entry(inode_lru.prev, struct inode, i_lru);/* +* we are inverting the inode_lru_lock/inode->i_lock here, +* so use a trylock. If we fail to get the lock, just move the +* inode to the back of the list so we don't spin on it. +*/ +if (!spin_trylock (&inode->i_lock)) { +list_move (&inode->i_lru, &inode_lru); +continue ; + } + +/* * Referenced or dirty inodes are still in use. Give them * another pass through the LRU as we canot reclaim them now. */ -spin_lock (&inode->i_lock);if (atomic_read (&inode->i_count) || (inode->i_state & ~I_REFERENCED )) {spin_unlock (&inode->i_lock);
@@ -652,17 +660,21 @@ static void prune_icache(int nr_to_scan)if (inode_has_buffers (inode) || inode->i_data.nrpages) {__iget (inode);spin_unlock (&inode->i_lock); +spin_unlock (&inode_lru_lock);spin_unlock (&inode_lock);if (remove_inode_buffers (inode)) reap +=invalidate_mapping_pages (&inode->i_data,0 ,-1 );iput (inode);spin_lock (&inode_lock); +spin_lock (&inode_lru_lock);if (inode !=list_entry (inode_lru.next, struct inode, i_lru))continue ;/* wrong inode or list_empty */ -spin_lock (&inode->i_lock); +/* avoid lock inversions with trylock */ +if (!spin_trylock (&inode->i_lock)) +continue ;if (!can_unuse (inode)) {spin_unlock (&inode->i_lock);continue ;
@@ -683,6 +695,7 @@ static void prune_icache(int nr_to_scan)__count_vm_events (KSWAPD_INODESTEAL , reap); else__count_vm_events (PGINODESTEAL ,reap ); +spin_unlock (&inode_lru_lock);spin_unlock (&inode_lock);dispose_list (&freeable);
Signature · 1 line
1.7.1
PATCH 4/4 fs: remove inode_lock from iput_final and prune_icache
In reply to Dave ChinnerFrom: Dave Chinner <[email protected]>
Now that inode state changes are protected by the inode->i_lock and the inode LRU manipulations by the inode_lru_lock, we can remove the inode_lock from prune_icache and the initial part of iput_final().
instead of using the inode_lock to protect the inode during iput_final, use the inode->i_lock instead. This protects the inode against new references being taken while we change the inode state to I_FREEING, as well as preventing prune_icache from grabbing the inode while we are manipulating it. Hence we no longer need the inode_lock in iput_final prior to setting I_FREEING on the inode.
For prune_icache, we no longer need the inode_lock to protect the LRU list, and the inodes themselves are protected against freeing races by the inode->i_lock. Hence we can lift the inode_lock from prune_icache as well.
--- fs/inode.c | 15 +++------------ 1 files changed, 3 insertions(+), 12 deletions(-)
fs/inode.c +3 -12
@@ -618,7 +618,6 @@ static void prune_icache(int nr_to_scan)unsigned long reap =0 ;down_read (&iprune_sem); -spin_lock (&inode_lock);spin_lock (&inode_lru_lock);for (nr_scanned =0 ; nr_scanned < nr_to_scan; nr_scanned++) {struct inode *inode;
@@ -644,8 +643,8 @@ static void prune_icache(int nr_to_scan)*/ if (atomic_read (&inode->i_count) || (inode->i_state & ~I_REFERENCED )) { -spin_unlock (&inode->i_lock);list_del_init (&inode->i_lru); +spin_unlock (&inode->i_lock);percpu_counter_dec (&nr_inodes_unused);continue ; }
@@ -653,20 +652,18 @@ static void prune_icache(int nr_to_scan)/* recently referenced inodes get one more pass */ if (inode->i_state &I_REFERENCED ) { inode->i_state &= ~I_REFERENCED ; -spin_unlock (&inode->i_lock);list_move (&inode->i_lru, &inode_lru); +spin_unlock (&inode->i_lock);continue ; } if (inode_has_buffers(inode) || inode->i_data.nrpages) { __iget(inode); spin_unlock(&inode->i_lock); spin_unlock(&inode_lru_lock); - spin_unlock(&inode_lock); if (remove_inode_buffers(inode)) reap += invalidate_mapping_pages(&inode->i_data, 0, -1); iput(inode); - spin_lock(&inode_lock); spin_lock(&inode_lru_lock); if (inode != list_entry(inode_lru.next,
@@ -696,7 +693,6 @@ static void prune_icache(int nr_to_scan)else __count_vm_events (PGINODESTEAL , reap);spin_unlock (&inode_lru_lock); -spin_unlock (&inode_lock);dispose_list (&freeable);up_read (&iprune_sem);
@@ -1413,7 +1409,6 @@ static void iput_final(struct inode *inode)const struct super_operations *op = inode->i_sb->s_op;int drop; -spin_lock (&inode->i_lock);WARN_ON (inode->i_state &I_NEW );if (op && op->drop_inode)
@@ -1426,16 +1421,13 @@ static void iput_final(struct inode *inode)if (!(inode->i_state & (I_DIRTY |I_SYNC )))inode_lru_list_add (inode);spin_unlock (&inode->i_lock); -spin_unlock (&inode_lock);return ; }if (!drop) { inode->i_state |=I_WILL_FREE ;spin_unlock (&inode->i_lock); -spin_unlock (&inode_lock);write_inode_now (inode,1 ); -spin_lock (&inode_lock);spin_lock (&inode->i_lock);WARN_ON (inode->i_state &I_NEW ); inode->i_state &= ~I_WILL_FREE ;
@@ -1444,7 +1436,6 @@ static void iput_final(struct inode *inode) inode->i_state |=I_FREEING ;inode_lru_list_del (inode);spin_unlock (&inode->i_lock); -spin_unlock (&inode_lock);evict (inode); }
@@ -1463,7 +1454,7 @@ void iput(struct inode *inode)if (inode) {BUG_ON (inode->i_state &I_CLEAR ); -if (atomic_dec_and_lock (&inode->i_count, &inode_lock)) +if (atomic_dec_and_lock (&inode->i_count, &inode->i_lock))iput_final (inode); } }
Signature · 1 line
1.7.1
Re: [PATCH 4/4] fs: remove inode_lock from iput_final and prune_icache
Looks good - although at this point you could already remove inode_lock from igrab, too.
Re: [PATCH 4/4] fs: remove inode_lock from iput_final and prune_icache
Quoted 2 lines · Christoph Hellwig
Looks good - although at this point you could already remove
inode_lockfrom igrab, too.
Yes, it probably can go at this point - after the next three patches it's pretty much the only place the lock is left, so it's not really protecting anything anymore...
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Quoted 5 lines · Dave Chinner
Hi Al,
The following patches break the inode LRU operations and the first half of
iput_final()out from under theinode_lock. The patches should apply to your current merge-stem tree.
The series looks good to me and survives QA so far.