fs/dcache: Limit # of negative dentries
Subsystems (4)
v2 cover fs/dcache: Limit # of negative dentries
v1->v2:
- Move the new nr_negative field to the end of dentry_stat_t structure as suggested by Matthew Wilcox.
- With the help of Miklos Szeredi, fix incorrect locking order in dentry_kill() by using lock_parent() instead of locking the parent's d_lock directly.
- Correctly account for positive to negative dentry transitions.
- Automatic pruning of negative dentries will now ignore the reference bit in negative dentries but not the regular shrinking.
A rogue application can potentially create a large number of negative dentries in the system consuming most of the memory available. This can impact performance of other applications running on the system.
This patchset introduces changes to the dcache subsystem to limit the number of negative dentries allowed to be created thus limiting the amount of memory that can be consumed by negative dentries.
Patch 1 tracks the number of negative dentries used and disallow the creation of more when the limit is reached.
Patch 2 enables /proc/sys/fs/dentry-state to report the number of negative dentries in the system.
Patch 3 enables automatic pruning of negative dentries when it is close to the limit so that we won't end up killing recently used negative dentries.
Patch 4 prevents racing between negative dentry pruning and umount operation.
Waiman Long (4):
fs/dcache: Limit numbers of negative dentries fs/dcache: Report negative dentry number in dentry-state fs/dcache: Enable automatic pruning of negative dentries fs/dcache: Protect negative dentry pruning from racing with umount
Documentation/admin-guide/kernel-parameters.txt | 7 + fs/dcache.c | 408 ++++++++++++++++++++++-- include/linux/dcache.h | 8 +- include/linux/list_lru.h | 1 + mm/list_lru.c | 4 +- 5 files changed, 392 insertions(+), 36 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v2 1/4 fs/dcache: Limit numbers of negative dentries
The number of positive dentries is limited by the number of files in the filesystems. The number of negative dentries, however, has no limit other than the total amount of memory available in the system. So a rogue application that generates a lot of negative dentries can potentially exhaust most of the memory available in the system impacting performance on other running applications.
To prevent this from happening, the dcache code is now updated to limit the amount of the negative dentries in the LRU lists that can be kept as a percentage of total available system memory. The default is 5% and can be changed by specifying the "neg_dentry_pc=" kernel command line option.
If the negative dentry limit is exceeded, newly created negative dentries will be killed right after use to avoid adding unpredictable latency to the directory lookup operation.
--- Documentation/admin-guide/kernel-parameters.txt | 7 + fs/dcache.c | 245 ++++++++++++++++++++---- include/linux/dcache.h | 1 + 3 files changed, 221 insertions(+), 32 deletions(-)
Documentation/admin-guide/kernel-parameters.txt +7 -0
@@ -2383,6 +2383,13 @@ n2= [NET] SDL Inc. RISCom/N2 synchronous serial card + neg_dentry_pc= [KNL] + Range: 1-50 + Default: 5 + This parameter specifies the amount of negative + dentries allowed in the system as a percentage of + total system memory. + netdev= [NET] Network devices parameters Format: <irq>,<io>,<mem_start>,<mem_end>,<name> Note that mem_start is often overloaded to mean
fs/dcache.c +213 -32
325 lines not on this page · raw message
include/linux/dcache.h +1 -0
@@ -218,6 +218,7 @@ struct dentry_operations {#define DCACHE_PAR_LOOKUP 0x10000000/* being looked up (with parent locked shared) */ #define DCACHE_DENTRY_CURSOR 0x20000000 +#define DCACHE_KILL_NEGATIVE 0x40000000/* Kill negative dentry */ extern seqlock_t rename_lock;
Signature · 1 line
1.8.3.1
PATCH v2 2/4 fs/dcache: Report negative dentry number in dentry-state
In reply to Waiman LongThe number of negative dentries currently in the system is now reported in the /proc/sys/fs/dentry-state file.
--- fs/dcache.c | 16 +++++++++++++++- include/linux/dcache.h | 7 ++++--- 2 files changed, 19 insertions(+), 4 deletions(-)
fs/dcache.c +15 -1
@@ -135,6 +135,7 @@ struct dentry_stat_t dentry_stat = {*/ #define NEG_DENTRY_BATCH (1 << 8)static long neg_dentry_percpu_limit__read_mostly ; +static long neg_dentry_nfree_init__read_mostly ;/* Free pool initial value */ static struct { raw_spinlock_t nfree_lock; long nfree; /* Negative dentry free pool */
@@ -176,11 +177,23 @@ static long get_nr_dentry_unused(void) return sum < 0 ? 0 : sum; } +static long get_nr_dentry_neg (void ) +{ +int i; +long sum =0 ; + +for_each_possible_cpu (i) + sum +=per_cpu (nr_dentry_neg, i); + sum += neg_dentry_nfree_init - ndblk.nfree; +return sum <0 ?0 : sum; +} +int proc_nr_dentry (struct ctl_table *table,int write,void __user *buffer,size_t *lenp,loff_t *ppos) { dentry_stat.nr_dentry =get_nr_dentry (); dentry_stat.nr_unused =get_nr_dentry_unused (); + dentry_stat.nr_negative =get_nr_dentry_neg ();return proc_doulongvec_minmax (table, write, buffer, lenp, ppos); }#endif
@@ -3733,7 +3746,8 @@ static void __init neg_dentry_init(void)raw_spin_lock_init (&ndblk.nfree_lock);/* 20% in global pool & 80% in percpu free */ - ndblk.nfree = totalram_pages * nr_dentry_page * neg_dentry_pc /500 ; + ndblk.nfree = neg_dentry_nfree_init + = totalram_pages * nr_dentry_page * neg_dentry_pc /500 ; cnt = ndblk.nfree *4 /num_possible_cpus ();if (unlikely (cnt <2 *NEG_DENTRY_BATCH )) cnt =2 *NEG_DENTRY_BATCH ;
include/linux/dcache.h +4 -3
@@ -63,9 +63,10 @@ struct qstr {struct dentry_stat_t {long nr_dentry;long nr_unused; -long age_limit;/* age in seconds */ -long want_pages;/* pages requested by system */ -long dummy[2 ]; +long age_limit;/* age in seconds */ +long want_pages;/* pages requested by system */ +long nr_negative;/* # of negative dentries */ +long dummy; };extern struct dentry_stat_t dentry_stat;
Signature · 1 line
1.8.3.1
PATCH v2 3/4 fs/dcache: Enable automatic pruning of negative dentries
In reply to Waiman LongHaving a limit for the number of negative dentries does have an undesirable side effect that no new negative dentries will be allowed when the limit is reached. This will have performance implication for some types of workloads.
So we need a way to prune the negative dentries so that new ones can be created. This is done by using the workqueue API to do the pruning gradually when a threshold is reached to minimize performance impact on other running tasks. The pruning is done at a frequency of 10 runs per second. Each run scans at most 256 LRU dentries for each node LRU list of a certain superblock. Some non-negative dentries that happen to be at the front of the LRU lists will also be pruned.
--- fs/dcache.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++ include/linux/list_lru.h | 1 + mm/list_lru.c | 4 +- 3 files changed, 117 insertions(+), 1 deletion(-)
fs/dcache.c +113 -0
@@ -134,13 +134,19 @@ struct dentry_stat_t dentry_stat = {* Macros and variables to manage and count negative dentries. */ #define NEG_DENTRY_BATCH (1 << 8) +#define NEG_PRUNING_DELAY (HZ/10)static long neg_dentry_percpu_limit__read_mostly ;static long neg_dentry_nfree_init__read_mostly ;/* Free pool initial value */ static struct {raw_spinlock_t nfree_lock;long nfree;/* Negative dentry free pool */ +struct super_block *prune_sb;/* Super_block for pruning */ +int neg_count, prune_count;/* Pruning counts */ } ndblk____cacheline_aligned_in_smp ; +static void prune_negative_dentry (struct work_struct *work); +static DECLARE_DELAYED_WORK(prune_neg_dentry_work, prune_negative_dentry); + static DEFINE_PER_CPU(long, nr_dentry); static DEFINE_PER_CPU(long, nr_dentry_unused); static DEFINE_PER_CPU(long, nr_dentry_neg);
@@ -323,6 +329,16 @@ static void __neg_dentry_inc(struct dentry *dentry)*/ if (!cnt) dentry->d_flags |=DCACHE_KILL_NEGATIVE ; + +/* +* Initiate negative dentry pruning if free pool has less than +* 1/4 of its initial value. +*/ +if (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/4 ) { +WRITE_ONCE (ndblk.prune_sb, dentry->d_sb); +schedule_delayed_work (&prune_neg_dentry_work, +NEG_PRUNING_DELAY ); + } }static inline void neg_dentry_inc (struct dentry *dentry)
@@ -1320,6 +1336,103 @@ void shrink_dcache_sb(struct super_block *sb) }EXPORT_SYMBOL (shrink_dcache_sb); +/* +* A modified version that attempts to remove a limited number of negative +* dentries as well as some other non-negative dentries at the front. +*/ +static enum lru_status dentry_negative_lru_isolate (struct list_head *item, +struct list_lru_one *lru,spinlock_t *lru_lock,void *arg) +{ +struct list_head *freeable = arg; + struct dentry *dentry = container_of(item, struct dentry, d_lru); +enum lru_status status =LRU_SKIP ; + +/* +* Stop further list walking for the current node list to limit +* performance impact, but allow further walking in the next node +* list. +*/ +if ((ndblk.neg_count >=NEG_DENTRY_BATCH ) || + (ndblk.prune_count >=NEG_DENTRY_BATCH )) { + ndblk.prune_count =0 ; +return LRU_STOP ; + } + +/* +* we are inverting the lru lock/dentry->d_lock here, +* so use a trylock. If we fail to get the lock, just skip +* it +*/ +if (!spin_trylock (&dentry->d_lock)) { + ndblk.prune_count++; +return LRU_SKIP ; + } + +/* +* Referenced dentries are still in use. If they have active +* counts, just remove them from the LRU. Otherwise give them +* another pass through the LRU. +*/ +if (dentry->d_lockref.count) { +d_lru_isolate (lru, dentry); + status =LRU_REMOVED ; + goto out; + } + +/* +* Dentries with reference bit on are moved back to the tail +* except for the negative ones. +*/ +if ((dentry->d_flags &DCACHE_REFERENCED ) && !d_is_negative (dentry)) { + dentry->d_flags &= ~DCACHE_REFERENCED ; + status =LRU_ROTATE ; + goto out; + } + + status =LRU_REMOVED ; +d_lru_shrink_move (lru, dentry, freeable); +if (d_is_negative (dentry)) + ndblk.neg_count++; +out: +spin_unlock (&dentry->d_lock); + ndblk.prune_count++; +return status; +} + +/* +* A workqueue function to prune negative dentry. +* +* The pruning is done gradually over time so as not to have noticeable +* performance impact. +*/ +static void prune_negative_dentry (struct work_struct *work) +{ +int freed; +struct super_block *sb =READ_ONCE (ndblk.prune_sb); +LIST_HEAD (dispose); + +if (!sb) +return ; + + ndblk.neg_count = ndblk.prune_count =0 ; + freed =list_lru_walk (&sb->s_dentry_lru, dentry_negative_lru_isolate, + &dispose,NEG_DENTRY_BATCH ); + +if (freed) +shrink_dentry_list (&dispose); +/* +* Continue delayed pruning once every second until negative dentry +* free pool is at least 1/2 of the initial value or the super_block +* has no more negative dentries left at the front. +*/ +if (ndblk.neg_count && + (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/2 )) +schedule_delayed_work (&prune_neg_dentry_work, +NEG_PRUNING_DELAY ); +else +WRITE_ONCE (ndblk.prune_sb,NULL ); +} +/** * enum d_walk_ret - action to talke during tree walk * @D_WALK_CONTINUE: contrinue walk
include/linux/list_lru.h +1 -0
@@ -22,6 +22,7 @@ enum lru_status {LRU_SKIP ,/* item cannot be locked, skip */ LRU_RETRY ,/* item not freeable. May drop the lock internally, but has to return locked. */ + LRU_STOP, /* stop walking the list */ }; struct list_lru_one {
mm/list_lru.c +3 -1
@@ -244,11 +244,13 @@ unsigned long list_lru_count_node(struct list_lru *lru, int nid)*/ assert_spin_locked (&nlru->lock); goto restart; +case LRU_STOP : + goto out;default :BUG (); } } - +out:spin_unlock (&nlru->lock);return isolated; }
Signature · 1 line
1.8.3.1
Re: [PATCH v2 3/4] fs/dcache: Enable automatic pruning of negative dentries
Quoted 4 lines · Waiman Long
Having a limit for the number of negative dentries does have an undesirable side effect that no new negative dentries will be allowed when the limit is reached. This will have performance implication for some types of workloads.
This really seems like a significant problem: negative dentries should be released in strict lru order because the chances are no-one cares about the least recently used one, but they may care about having the most recently created one.
[...]
Quoted 9 lines
fs/dcache.c +3
-0@@ -323,6 +329,16 @@ static void __neg_dentry_inc(struct dentry*dentry) */ if (!cnt) dentry->d_flags |= DCACHE_KILL_NEGATIVE; + +/* +* Initiate negative dentry pruning if free pool has less than
fs/dcache.c static void __neg_dentry_inc(struct dentry *dentry) · line 341
Quoted 7 lines
fs/dcache.c +7
-0+* 1/4 of its initial value. +*/ +if (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/4 ) { +WRITE_ONCE (ndblk.prune_sb, dentry->d_sb); +schedule_delayed_work (&prune_neg_dentry_work, + NEG_PRUNING_DELAY); + }
So here, why not run the negative dentry shrinker synchronously to see if we can shrink the cache and avoid killing the current negative dentry. If there are context problems doing that, we should at least make the effort to track down the least recently used negative dentry and mark that for killing instead.
James
Re: [PATCH v2 3/4] fs/dcache: Enable automatic pruning of negative dentries
Quoted 9 lines · James Bottomley
Quoted 4 lines · Waiman Long
Having a limit for the number of negative dentries does have an undesirable side effect that no new negative dentries will be allowed when the limit is reached. This will have performance implication for some types of workloads.
This really seems like a significant problem: negative dentries should be released in strict lru order because the chances are no-one cares about the least recently used one, but they may care about having the most recently created one.
This should not happen under normal circumstances as the asynchronous shrinker should be able to keep enough free negative dentry available in the pool that direct negative dentry killing will rarely happen.
Quoted 14 lines
[...]
Quoted 13 lines
fs/dcache.c +3
-0@@ -323,6 +329,16 @@ static void __neg_dentry_inc(struct dentry*dentry) */ if (!cnt) dentry->d_flags |=DCACHE_KILL_NEGATIVE ; + +/* +* Initiate negative dentry pruning if free pool has less than
fs/dcache.c +4
-0+* 1/4 of its initial value. +*/ +if (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/4 ) { +WRITE_ONCE (ndblk.prune_sb, dentry->d_sb);
Quoted 8 lines
fs/dcache.c static void __neg_dentry_inc(struct dentry *dentry) · line 341
Quoted 3 lines
fs/dcache.c +3
-0+schedule_delayed_work (&prune_neg_dentry_work, +NEG_PRUNING_DELAY ); + }So here, why not run the negative dentry shrinker synchronously to see if we can shrink the cache and avoid killing the current negative dentry. If there are context problems doing that, we should at least make the effort to track down the least recently used negative dentry and mark that for killing instead.
Only one CPU will be calling the asynchronous shrinker. So its effect on the overall performance of the system should be negligible.
Allowing all CPUs to potentially do synchronous shrinking can cause a lot of lock and cacheline contention. I will look further to see if there is opportunity to do some optimistic synchronous shrinking. If that fails because of a contended lock, for example, we will need to fall back to killing the dentry. That should only happen under the worst case situation, like when a malicious process is running.
Cheers,
Longman
Re: [PATCH v2 3/4] fs/dcache: Enable automatic pruning of negative dentries
Quoted 8 lines · Waiman Long
Quoted 7 lines · James Bottomley
Quoted 5 lines · Waiman Long
Having a limit for the number of negative dentries does have an undesirable side effect that no new negative dentries will be allowed when the limit is reached. This will have performance implication for some types of workloads.
Quoted 8 lines
Quoted 4 lines
This really seems like a significant problem: negative dentries should be released in strict lru order because the chances are no- one cares about the least recently used one, but they may care about having the most recently created one.
This should not happen under normal circumstances as the asynchronous shrinker should be able to keep enough free negative dentry available in the pool that direct negative dentry killing will rarely happen.
But that's an argument for not bothering with the patch set at all: if it's a corner case that rarely occurs.
Perhaps we should start with why the series is important and then get back to what we do if the condition is hit?
Quoted 26 lines
Quoted 17 lines
[...]
Quoted 15 lines
fs/dcache.c +3
-0@@ -323,6 +329,16 @@ static void __neg_dentry_inc(struct dentry*dentry) */ if (!cnt) dentry->d_flags |= DCACHE_KILL_NEGATIVE; + +/* +* Initiate negative dentry pruning if free pool has less
thanfs/dcache.c +4
-0+* 1/4 of its initial value. +*/ +if (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/4 ) { +WRITE_ONCE (ndblk.prune_sb, dentry->d_sb);Quoted 8 lines
fs/dcache.c static void __neg_dentry_inc(struct dentry *dentry) · line 341
Quoted 3 lines
fs/dcache.c +3
-0+schedule_delayed_work (&prune_neg_dentry_work, + NEG_PRUNING_DELAY); + }So here, why not run the negative dentry shrinker synchronously to see if we can shrink the cache and avoid killing the current negative dentry. If there are context problems doing that, we should at least make the effort to track down the least recently used negative dentry and mark that for killing instead.
Quoted 5 lines
Only one CPU will be calling the asynchronous shrinker. So its effect on the overall performance of the system should be negligible.
Allowing all CPUs to potentially do synchronous shrinking can cause a lot of lock and cacheline contention.
Does that matter on something you thought was a rare occurrence above? Plus, if the only use case is malicious user, as you say below, then they get to pay the biggest overhead penalty because they're the ones trying to create negative dentries.
Perhaps if the threat model truly is malicious users creating negative dentries then a better fix is to add a delay penalty to true negative dentry creation (say short msleep) when we get into this situation (if you only pay the penalty on true negative dentry creation, not negative promotes to positive, then we'll mostly penalise the process trying to be malicious).
Quoted 5 lines
I will look further to see if there is opportunity to do some
optimistic synchronous shrinking. If that fails because of a contended lock, for example, we will need to fall back to killing the dentry. That should only happen under the worst case situation, like when a malicious process is running.
Right, so I can force this by doing a whole load of non existent file lookups then what happens: negative dentries stop working for everyone because they're killed as soon as they're created. Negative dentries are useful performance enhancements for things like the usual does x.lock exist, if not modify x things that applications do.
It also looks like the dentry never loses DCACHE_KILL_NEGATIVE, so if I'm creating the x.lock file, and we're in this situation, it gets a positive dentry with the DCACHE_KILL_NEGATIVE flag set (because we start the lookup finding a negative dentry which gets the flag and then promote it to positive).
James
Re: [PATCH v2 3/4] fs/dcache: Enable automatic pruning of negative dentries
Quoted 11 lines · James Bottomley
Quoted 10 lines · Waiman Long
Quoted 9 lines · James Bottomley
Quoted 4 lines · Waiman Long
Having a limit for the number of negative dentries does have an undesirable side effect that no new negative dentries will be allowed when the limit is reached. This will have performance implication for some types of workloads.
This really seems like a significant problem: negative dentries should be released in strict lru order because the chances are no- one cares about the least recently used one, but they may care about having the most recently created one.
Quoted 8 lines
Quoted 3 lines
This should not happen under normal circumstances as the asynchronous shrinker should be able to keep enough free negative dentry available in the pool that direct negative dentry killing will rarely happen.
But that's an argument for not bothering with the patch set at all: if it's a corner case that rarely occurs.
Perhaps we should start with why the series is important and then get back to what we do if the condition is hit?
There is concern that unlimited negative dentry growth can be used as a kind of DoS attack by robbing system of all its memory and triggering massive memory reclaim or even OOM kill. This patchset is aimed to prevent this kind of worst case situation from happening.
Quoted 33 lines
Quoted 23 lines
Quoted 15 lines
[...]
Quoted 14 lines
fs/dcache.c +3
-0@@ -323,6 +329,16 @@ static void __neg_dentry_inc(struct dentry*dentry) */ if (!cnt) dentry->d_flags |=DCACHE_KILL_NEGATIVE ; + +/* +* Initiate negative dentry pruning if free pool has less
thanfs/dcache.c +4
-0+* 1/4 of its initial value. +*/ +if (READ_ONCE (ndblk.nfree) < neg_dentry_nfree_init/4 ) { +WRITE_ONCE (ndblk.prune_sb, dentry->d_sb);Quoted 8 lines
fs/dcache.c static void __neg_dentry_inc(struct dentry *dentry) · line 341
Quoted 3 lines
fs/dcache.c +3
-0+schedule_delayed_work (&prune_neg_dentry_work, +NEG_PRUNING_DELAY ); + }So here, why not run the negative dentry shrinker synchronously to see if we can shrink the cache and avoid killing the current negative dentry. If there are context problems doing that, we should at least make the effort to track down the least recently used negative dentry and mark that for killing instead.
Quoted 5 lines
Only one CPU will be calling the asynchronous shrinker. So its effect on the overall performance of the system should be negligible.
Allowing all CPUs to potentially do synchronous shrinking can cause a lot of lock and cacheline contention.
Does that matter on something you thought was a rare occurrence above?
Plus, if the only use case is malicious user, as you say below, then
they get to pay the biggest overhead penalty because they're the ones trying to create negative dentries.
Quoted 6 lines
Perhaps if the threat model truly is malicious users creating negative dentries then a better fix is to add a delay penalty to true negative dentry creation (say short msleep) when we get into this situation (if you only pay the penalty on true negative dentry creation, not negative promotes to positive, then we'll mostly penalise the process trying to be malicious).
I can insert a delay loop before killing the negative dentry. Taking a msleep can be problematic as the callers of dput() may not be in a sleepable state.
Quoted 10 lines
Quoted 5 lines
I will look further to see if there is opportunity to do some
optimistic synchronous shrinking. If that fails because of a contended lock, for example, we will need to fall back to killing the dentry. That should only happen under the worst case situation, like when a malicious process is running.Right, so I can force this by doing a whole load of non existent file lookups then what happens: negative dentries stop working for everyone because they're killed as soon as they're created. Negative dentries are useful performance enhancements for things like the usual does x.lock exist, if not modify x things that applications do.
I am well aware of the performance benefit offered by negative dentries. That is why I added this patch to prune the LRU list before it is too late and is forced to kill negative dentries. However, negative dentry killing may still happen if the negative dentry generation rate is faster than the pruning rate.
This can be caused by bugs in the applications or malicious intent. If this happens, it is likely that negative dentries will consume most of the system memory without this patchset. Application performance will suffer somewhat, but it will not as bad as when most of the memory are consumed by negative dentries.
Quoted 5 lines
It also looks like the dentry never loses
DCACHE_KILL_NEGATIVE, so if I'm creating the x.lock file, and we're in this situation, it gets a positive dentry with theDCACHE_KILL_NEGATIVEflag set (because we start the lookup finding a negative dentry which gets the flag and then promote it to positive).
Thanks for pointing this out. I am going to fix this bug in my patch.
Cheers,
Longman
PATCH v2 4/4 fs/dcache: Protect negative dentry pruning from racing with umount
In reply to Waiman LongThe negative dentry pruning is done on a specific super_block set in the ndblk.prune_sb variable. If the super_block is also being un-mounted concurrently, the content of the super_block may no longer be valid.
To protect against such racing condition, a new lock is added to the ndblk structure to synchronize the negative dentry pruning and umount operation. This is a regular spinlock as the pruning operation can be quite time consuming.
--- fs/dcache.c | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-)
fs/dcache.c +39 -3
@@ -139,11 +139,13 @@ struct dentry_stat_t dentry_stat = {static long neg_dentry_nfree_init__read_mostly ;/* Free pool initial value */ static struct {raw_spinlock_t nfree_lock; +spinlock_t prune_lock;/* Lock for protecting pruning */ long nfree;/* Negative dentry free pool */ struct super_block *prune_sb;/* Super_block for pruning */ int neg_count, prune_count;/* Pruning counts */ } ndblk____cacheline_aligned_in_smp ; +static void clear_prune_sb_for_umount (struct super_block *sb);static void prune_negative_dentry (struct work_struct *work); static DECLARE_DELAYED_WORK(prune_neg_dentry_work, prune_negative_dentry);
@@ -1323,6 +1325,7 @@ void shrink_dcache_sb(struct super_block *sb) {long freed; +clear_prune_sb_for_umount (sb); do { LIST_HEAD(dispose);
@@ -1353,7 +1356,8 @@ static enum lru_status dentry_negative_lru_isolate(struct list_head *item,* list. */ if ((ndblk.neg_count >=NEG_DENTRY_BATCH ) || - (ndblk.prune_count >=NEG_DENTRY_BATCH )) { + (ndblk.prune_count >=NEG_DENTRY_BATCH ) || + !READ_ONCE (ndblk.prune_sb)) { ndblk.prune_count =0 ;return LRU_STOP ; }
@@ -1408,15 +1412,24 @@ static enum lru_status dentry_negative_lru_isolate(struct list_head *item,static void prune_negative_dentry (struct work_struct *work) {int freed; -struct super_block *sb =READ_ONCE (ndblk.prune_sb); +struct super_block *sb;LIST_HEAD (dispose); -if (!sb) +/* +* The prune_lock is used to protect negative dentry pruning from +* racing with concurrent umount operation. +*/ +spin_lock (&ndblk.prune_lock); + sb =READ_ONCE (ndblk.prune_sb); +if (!sb) { +spin_unlock (&ndblk.prune_lock);return ; + } ndblk.neg_count = ndblk.prune_count =0 ; freed =list_lru_walk (&sb->s_dentry_lru, dentry_negative_lru_isolate, &dispose,NEG_DENTRY_BATCH ); +spin_unlock (&ndblk.prune_lock);if (freed)shrink_dentry_list (&dispose);
@@ -1433,6 +1446,27 @@ static void prune_negative_dentry(struct work_struct *work)WRITE_ONCE (ndblk.prune_sb,NULL ); } +/* +* This is called before an umount to clear ndblk.prune_sb if it +* matches the given super_block. +*/ +static void clear_prune_sb_for_umount (struct super_block *sb) +{ +if (likely (READ_ONCE (ndblk.prune_sb) != sb)) +return ; +WRITE_ONCE (ndblk.prune_sb,NULL ); +/* +* Need to wait until an ongoing pruning operation, if present, +* is completed. +* +* Clearing ndblk.prune_sb will hasten the completion of pruning. +* In the unlikely event that ndblk.prune_sb is set to another +* super_block, the waiting will last the complete pruning operation +* which shouldn't be that long either. +*/ +spin_unlock_wait (&ndblk.prune_lock); +} +/** * enum d_walk_ret - action to talke during tree walk * @D_WALK_CONTINUE: contrinue walk
@@ -1755,6 +1789,7 @@ void shrink_dcache_for_umount(struct super_block *sb)WARN (down_read_trylock (&sb->s_umount),"s_umount should've been locked" ); +clear_prune_sb_for_umount (sb); dentry = sb->s_root; sb->s_root =NULL ;do_one_tree (dentry);
@@ -3857,6 +3892,7 @@ static void __init neg_dentry_init(void)unsigned long cnt;raw_spin_lock_init (&ndblk.nfree_lock); +spin_lock_init (&ndblk.prune_lock);/* 20% in global pool & 80% in percpu free */ ndblk.nfree = neg_dentry_nfree_init
Signature · 1 line
1.8.3.1