cifs: Add a laundromat thread for cached directories
Subsystems (2)
PATCH 1/1 cifs: Add a laundromat thread for cached directories
and drop cached directories after 30 seconds
--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)
fs/smb/client/cached_dir.c +67 -0
@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { +if (jiffies > cfid->time +HZ *30 ) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;
@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }
@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;
fs/smb/client/cached_dir.h +1 -0
@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void );
Signature · 1 line
2.35.3
updated to fix two checkpatch warnings (see attached) and tentatively merged into for-next pending review/testing
Quoted 116 lines · Ronnie Sahlberg
and drop cached directories after 30 seconds
Signed-off-by Ronnie Sahlberg <[email protected]>--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)fs/smb/client/cached_dir.c +67
-0@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { +if (jiffies > cfid->time +HZ *30 ) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;fs/smb/client/cached_dir.h +1
-1@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void ); --2.35.3
Signature · 3 lines
Thanks, Steve
From cc9ed26848f724065aba50647540e5ebe6e0d3e9 Mon Sep 17 00:00:00 2001
From: Ronnie Sahlberg <[email protected]>
Date: Thu, 6 Jul 2023 12:32:24 +1000
Subject: [PATCH] cifs: Add a laundromat thread for cached directories
and drop cached directories after 30 seconds
--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)
fs/smb/client/cached_dir.c +67 -0
@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { + if (time_after(jiffies, cfid->time + HZ * 30) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;
@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }
@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;
fs/smb/client/cached_dir.h +1 -0
@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void );
Signature · 1 line
2.34.1
Quoted 127 lines · Steve French
updated to fix two checkpatch warnings (see attached) and tentatively merged into for-next pending review/testing
Quoted 114 lines · Ronnie Sahlberg
and drop cached directories after 30 seconds
Signed-off-by Ronnie Sahlberg <[email protected]>--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)fs/smb/client/cached_dir.c +67
-0@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { +if (jiffies > cfid->time +HZ *30 ) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;fs/smb/client/cached_dir.h +1
-1@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void ); --Quoted 2 lines
2.35.3
--
Thanks,
Quoted 1 line
Steve
Hi Ronnie,
Why do we need this?
We only cache dirents if we have the parent directory lease. As long as we have the lease, why can't we continue to cache these?
Signature · 2 lines
Regards, Shyam
ronnie sahlberg
reviewerWe can only cache a limited amount of entries.
We need to force entries to be dropped from the cache at regular intervals to make room for potential new entries/different directories.
Access patterns change over time and what is the "hot" directory will also change over time so we need to drop entries to make sure that when some directory becomes hot there will be decent chance that it will be able to become cached.
If a directory becomes "cold" we no longer want it to take up entries in our cache.
regards
ronnie sahlberg
Quoted 140 lines · Shyam Prasad N
Quoted 127 lines · Steve French
updated to fix two checkpatch warnings (see attached) and tentatively merged into for-next pending review/testing
Quoted 114 lines · Ronnie Sahlberg
and drop cached directories after 30 seconds
Signed-off-by Ronnie Sahlberg <[email protected]>--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)fs/smb/client/cached_dir.c +67
-0@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { +if (jiffies > cfid->time +HZ *30 ) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;fs/smb/client/cached_dir.h +1
-1@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void ); --Quoted 2 lines
2.35.3
--
Thanks,Quoted 1 line
Steve
Hi Ronnie,
Why do we need this?
We only cache dirents if we have the parent directory lease. As long as we have the lease, why can't we continue to cache these?--
Regards,
Shyam
Quoted 6 lines · ronnie sahlberg
We can only cache a limited amount of entries.
We need to force entries to be dropped from the cache at regular intervals to make room for potential new entries/different directories.
Quoted 8 lines
Access patterns change over time and what is the "hot" directory will also change over time so we need to drop entries to make sure that when some directory becomes hot there will be decent chance that it will be able to become cached.
If a directory becomes "cold" we no longer want it to take up entries in our cache.
Makes sense.
However, the value of MAX_CACHED_FIDS to 16 seems very restrictive. And as Steve suggested, 30s expiry seems very aggressive. I think we can increase both.
Also, I was wondering how do we handle cleanup of cfids when VFS decides to free the inode (maybe due to memory crunch?).
Regards,
Shyam
Quoted 145 lines
regards
ronnie sahlbergQuoted 140 lines · Shyam Prasad N
Quoted 127 lines · Steve French
updated to fix two checkpatch warnings (see attached) and tentatively merged into for-next pending review/testing
Quoted 114 lines · Ronnie Sahlberg
and drop cached directories after 30 seconds
Signed-off-by Ronnie Sahlberg <[email protected]>--- fs/smb/client/cached_dir.c | 67 ++++++++++++++++++++++++++++++++++++++ fs/smb/client/cached_dir.h | 1 + 2 files changed, 68 insertions(+)fs/smb/client/cached_dir.c +67
-0@@ -568,6 +568,53 @@ static void free_cached_dir(struct cached_fid *cfid)kfree (cfid); } +static int +cifs_cfids_laundromat_thread (void *p) +{ +struct cached_fids *cfids = p; +struct cached_fid *cfid, *q; +struct list_head entry; + +while (!kthread_should_stop ()) { +ssleep (1 ); +INIT_LIST_HEAD (&entry); +if (kthread_should_stop ()) +return 0 ; +spin_lock (&cfids->cfid_list_lock); +list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { +if (jiffies > cfid->time +HZ *30 ) { +list_del (&cfid->entry); +list_add (&cfid->entry, &entry); + cfids->num_entries--; + } + } +spin_unlock (&cfids->cfid_list_lock); + +list_for_each_entry_safe (cfid, q, &entry, entry) { + cfid->on_list = false; +list_del (&cfid->entry); +/* +* Cancel, and wait for the work to finish in +* case we are racing with it. +*/ +cancel_work_sync (&cfid->lease_break); +if (cfid->has_lease) { +/* +* We lease has not yet been cancelled from +* the server so we need to drop the reference. +*/ +spin_lock (&cfids->cfid_list_lock); + cfid->has_lease = false; +spin_unlock (&cfids->cfid_list_lock); +kref_put (&cfid->refcount, smb2_close_cached_fid); + } + } + } + +return 0 ; +} + +struct cached_fids *init_cached_dirs (void ) {struct cached_fids *cfids;@@ -577,6 +624,20 @@ struct cached_fids *init_cached_dirs(void)return NULL ;spin_lock_init (&cfids->cfid_list_lock);INIT_LIST_HEAD (&cfids->entries); + +/* +* since we're in a cifs function already, we know that +* this will succeed. No need for try_module_get(). +*/ +__module_get (THIS_MODULE ); + cfids->laundromat =kthread_run (cifs_cfids_laundromat_thread, + cfids,"cifsd-cfid-laundromat" ); +if (IS_ERR (cfids->laundromat)) { +cifs_dbg (VFS ,"Failed to start cfids laundromat thread.\n" ); +kfree (cfids); +module_put (THIS_MODULE ); +return NULL ; + }return cfids; }@@ -589,6 +650,12 @@ void free_cached_dirs(struct cached_fids *cfids)struct cached_fid *cfid, *q;LIST_HEAD (entry); +if (cfids->laundromat) { +kthread_stop (cfids->laundromat); + cfids->laundromat =NULL ; +module_put (THIS_MODULE ); + } +spin_lock (&cfids->cfid_list_lock);list_for_each_entry_safe (cfid, q, &cfids->entries, entry) { cfid->on_list = false;fs/smb/client/cached_dir.h +1
-1@@ -57,6 +57,7 @@ struct cached_fids {spinlock_t cfid_list_lock;int num_entries;struct list_head entries; +struct task_struct *laundromat; };extern struct cached_fids *init_cached_dirs (void ); --Quoted 2 lines
2.35.3
--
Thanks,Quoted 1 line
Steve
Hi Ronnie,
Why do we need this?
We only cache dirents if we have the parent directory lease. As long as we have the lease, why can't we continue to cache these?--
Regards,
Shyam
Signature · 2 lines
Regards, Shyam
ronnie sahlberg
reviewerQuoted 18 lines · Shyam Prasad N
Quoted 6 lines · ronnie sahlberg
We can only cache a limited amount of entries.
We need to force entries to be dropped from the cache at regular intervals to make room for potential new entries/different directories.Quoted 8 lines
Access patterns change over time and what is the "hot" directory will also change over time so we need to drop entries to make sure that when some directory becomes hot there will be decent chance that it will be able to become cached.
If a directory becomes "cold" we no longer want it to take up entries in our cache.
Quoted 5 lines
Makes sense.
However, the value of
MAX_CACHED_FIDSto 16 seems very restrictive. And as Steve suggested, 30s expiry seems very aggressive. I think we can increase both.
We can and should increase it at some stage but lets keep it small and short for now until we feel confident it is all stable. Once we are confident it is good we can increase these quite a bit.
Quoted 18 lines · Shyam Prasad N
Quoted 6 lines · ronnie sahlberg
We can only cache a limited amount of entries.
We need to force entries to be dropped from the cache at regular intervals to make room for potential new entries/different directories.Quoted 8 lines
Access patterns change over time and what is the "hot" directory will also change over time so we need to drop entries to make sure that when some directory becomes hot there will be decent chance that it will be able to become cached.
If a directory becomes "cold" we no longer want it to take up entries in our cache.
Quoted 5 lines
Makes sense.
However, the value of
MAX_CACHED_FIDSto 16 seems very restrictive. And as Steve suggested, 30s expiry seems very aggressive. I think we can increase both.
At least for the short term we can make this configurable (which will make it easier to test the best default values in the long run) e.g.
via mount parm. Will make it much easier to experiment to optimize
the value
Signature · 3 lines
Thanks, Steve