CacheFiles: Provide read-and-reset release counters for cachefilesd
Subsystems (4)
David Howells
maintainerPATCH 1/1 CacheFiles: Provide read-and-reset release counters for cachefilesd
Provide read-and-reset objects- and blocks-released counters for cachefilesd to use to work out whether there's anything new that can be culled.
One of the problems cachefilesd has is that if all the objects in the cache are pinned by inodes lying dormant in the kernel inode cache, there isn't anything for it to cull. In such a case, it just spins around walking the filesystem tree and scanning for something to cull. This eats up a lot of CPU time.
By telling cachefilesd if there have been any releases, the daemon can sleep until there is the possibility of something to do.
cachefilesd finds this information by the following means:
(1) When the control fd is read, the kernel presents a list of values of interest. "freleased=N" and "breleased=N" are added to this list to indicate the number of files released and number of blocks released since the last read call. At this point the counters are reset.
(2) POLLIN is signalled if the number of files released becomes greater
than 0.
Note that by 'released' it just means that the kernel has released its interest in those files for the moment, not necessarily that the files should be deleted from the cache.
---
fs/cachefiles/daemon.c | 13 ++++++++++--- fs/cachefiles/interface.c | 11 ++--------- fs/cachefiles/internal.h | 4 ++++ fs/cachefiles/namei.c | 28 +++++++++++++++++++++++----- 4 files changed, 39 insertions(+), 17 deletions(-)
fs/cachefiles/daemon.c +10 -3
@@ -162,6 +162,8 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, size_t buflen, loff_t *pos) { struct cachefiles_cache *cache = file->private_data; +unsigned long long b_released; +unsigned f_released;char buffer[256 ];int n;
@@ -174,6 +176,8 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer,cachefiles_has_space (cache,0 ,0 );/* summarise */ + f_released =atomic_xchg (&cache->f_released,0 ); + b_released =atomic_long_xchg (&cache->b_released,0 );clear_bit (CACHEFILES_STATE_CHANGED , &cache->flags); n = snprintf(buffer, sizeof(buffer),
@@ -183,15 +187,18 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer," fstop=%llx" " brun=%llx" " bcull=%llx" -" bstop=%llx" , +" bstop=%llx" +" freleased=%x" +" breleased=%llx" ,test_bit (CACHEFILES_CULLING , &cache->flags) ?'1' :'0' , (unsigned long long ) cache->frun, (unsigned long long ) cache->fcull, (unsigned long long ) cache->fstop, (unsigned long long ) cache->brun, (unsigned long long ) cache->bcull, - (unsigned long long ) cache->bstop - ); + (unsigned long long ) cache->bstop, + f_released, + b_released);if (n > buflen)return -EMSGSIZE ;
fs/cachefiles/interface.c +2 -9
@@ -291,15 +291,8 @@ static void cachefiles_drop_object(struct fscache_object *_object) }/* note that the object is now inactive */ -if (test_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags)) { -write_lock (&cache->active_lock); -if (!test_and_clear_bit (CACHEFILES_OBJECT_ACTIVE , - &object->flags)) -BUG (); -rb_erase (&object->active_node, &cache->active_nodes); -wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); -write_unlock (&cache->active_lock); - } +if (test_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags)) +cachefiles_mark_object_inactive (cache, object);dput (object->dentry); object->dentry =NULL ;
fs/cachefiles/internal.h +4 -0
@@ -66,6 +66,8 @@ struct cachefiles_cache {struct rb_root active_nodes;/* active nodes (can't be culled) */ rwlock_t active_lock;/* lock for active_nodes */ atomic_t gravecounter;/* graveyard uniquifier */ +atomic_t f_released;/* number of objects released lately */ +atomic_long_t b_released;/* number of blocks released lately */ unsigned frun_percent;/* when to stop culling (% files) */ unsigned fcull_percent;/* when to start culling (% files) */ unsigned fstop_percent;/* when to stop allocating (% files) */
@@ -157,6 +159,8 @@ extern char *cachefiles_cook_key(const u8 *raw, int keylen, uint8_t type);/* * namei.c */ +extern void cachefiles_mark_object_inactive (struct cachefiles_cache *cache, +struct cachefiles_object *object);extern int cachefiles_delete_object (struct cachefiles_cache *cache,struct cachefiles_object *object); extern int cachefiles_walk_to_object(struct cachefiles_object *parent,
fs/cachefiles/namei.c +23 -5
@@ -258,6 +258,28 @@ requeue: }/* +* Mark an object as being inactive. +*/ +void cachefiles_mark_object_inactive (struct cachefiles_cache *cache, +struct cachefiles_object *object) +{ +write_lock (&cache->active_lock); +rb_erase (&object->active_node, &cache->active_nodes); +clear_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags); +write_unlock (&cache->active_lock); + +wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); + +/* This object can now be culled, so we need to let the daemon know +* that there is something it can remove if it needs to. +*/ +atomic_long_add (d_backing_inode (object->dentry)->i_blocks, + &cache->b_released); +if (atomic_inc_return (&cache->f_released)) +cachefiles_state_changed (cache); +} + +/* * delete an object representation from the cache * - file backed objects are unlinked * - directory backed objects are stuffed into the graveyard for userspace to
@@ -684,11 +706,7 @@ mark_active_timed_out: check_error:_debug ("check error %d" , ret); -write_lock (&cache->active_lock); -rb_erase (&object->active_node, &cache->active_nodes); -clear_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags); -wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); -write_unlock (&cache->active_lock); +cachefiles_mark_object_inactive (cache, object); release_dentry:dput (object->dentry); object->dentry =NULL ;
David Howells
maintainerPATCH 1/2 cachefilesd can spin when disk space is short.
From: NeilBrown <[email protected]>
When cachefilesd finds that it needs to cull, but that culling doesn't achieve anything, it sets an alarm to wake it in 30 seconds to try again. But as read_cache_state() will detect that culling is still needed, it will immediately try again anyway.
This results in 100% cpu usage of no value.
This patch causes culling to be blocked until the 30 second alarm goes off.
It also changes the test to decide whether to enter poll() after blocking signals to test exactly those values that might be changed by a signal. Testing these is important, testing anything else is pointless.
---
cachefilesd.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
cachefilesd.c +6 -3
@@ -99,6 +99,7 @@ static struct object **cullready;static unsigned nr_in_build_table;static unsigned nr_in_ready_table;static int ncullable; +static bool cull_delayed;static const char *configfile ="/etc/cachefilesd.conf" ;
@@ -246,6 +247,7 @@ static void sigio(int sig)static void sigalrm (int sig) { jumpstart_scan = true; + cull_delayed = false; }/*****************************************************************************/
@@ -613,11 +615,11 @@ static void cachefilesd(void)/* sleep without racing on reap and cull with the signal * handlers */ -if (!scan && !reap && !cull) { +if (!scan && !reap && !(cull && !cull_delayed)) {if (sigprocmask (SIG_BLOCK , &sigs, &osigs) <0 )oserror ("Unable to block signals" ); -if (!reap && !cull) { +if (!reap && !stop && !jumpstart_scan) {if (ppoll (pollfds,1 ,NULL , &osigs) <0 && errno !=EINTR )oserror ("Unable to suspend process" );
@@ -644,7 +646,7 @@ static void cachefilesd(void)if (cull) {if (nr_in_ready_table >0 )cull_objects (); -else if (nr_in_build_table ==0 ) +else if (nr_in_build_table ==0 && !cull_delayed) jumpstart_scan = true; }
@@ -1364,6 +1366,7 @@ static void decant_cull_table(void)/* if nothing there, scan again in a short while */ if (nr_in_build_table ==0 ) { + cull_delayed = true;signal (SIGALRM , sigalrm);alarm (30 );return ;
Re: [Linux-cachefs] [PATCH 1/2] cachefilesd can spin when disk space is short.
Quoted 9 lines · David Howells
From: NeilBrown <[email protected]>
When cachefilesd finds that it needs to cull, but that culling doesn't achieve anything, it sets an alarm to wake it in 30 seconds to try again. But as
read_cache_state()will detect that culling is still needed, it will immediately try again anyway.This results in 100% cpu usage of no value.
Quoted 8 lines
This patch causes culling to be blocked until the 30 second alarm goes off.
It also changes the test to decide whether to enter
poll()after blocking signals to test exactly those values that might be changed by a signal. Testing these is important, testing anything else is pointless.Signed-off-by NeilBrown <[email protected]>Signed-off-by David Howells <[email protected]>
steved.
Quoted 57 lines
---
cachefilesd.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)cachefilesd.c +6
-3@@ -99,6 +99,7 @@ static struct object **cullready;static unsigned nr_in_build_table;static unsigned nr_in_ready_table;static int ncullable; +static bool cull_delayed;static const char *configfile ="/etc/cachefilesd.conf" ;@@ -246,6 +247,7 @@ static void sigio(int sig)static void sigalrm (int sig) { jumpstart_scan = true; + cull_delayed = false; }/*****************************************************************************/ @@ -613,11 +615,11 @@ static void cachefilesd(void)/* sleep without racing on reap and cull with the signal * handlers */ -if (!scan && !reap && !cull) { +if (!scan && !reap && !(cull && !cull_delayed)) {if (sigprocmask (SIG_BLOCK , &sigs, &osigs) <0 )oserror ("Unable to block signals" ); -if (!reap && !cull) { +if (!reap && !stop && !jumpstart_scan) {if (ppoll (pollfds,1 ,NULL , &osigs) <0 && errno !=EINTR )oserror ("Unable to suspend process" );@@ -644,7 +646,7 @@ static void cachefilesd(void)if (cull) {if (nr_in_ready_table >0 )cull_objects (); -else if (nr_in_build_table ==0 ) +else if (nr_in_build_table ==0 && !cull_delayed) jumpstart_scan = true; }@@ -1364,6 +1366,7 @@ static void decant_cull_table(void)/* if nothing there, scan again in a short while */ if (nr_in_build_table ==0 ) { + cull_delayed = true;signal (SIGALRM , sigalrm);alarm (30 );return ;
Re: [Linux-cachefs] [PATCH 1/2] cachefilesd can spin when disk space is short.
In reply to David HowellsQuoted 9 lines · David Howells
From: NeilBrown <[email protected]>
When cachefilesd finds that it needs to cull, but that culling doesn't achieve anything, it sets an alarm to wake it in 30 seconds to try again. But as
read_cache_state()will detect that culling is still needed, it will immediately try again anyway.This results in 100% cpu usage of no value.
Quoted 8 lines
This patch causes culling to be blocked until the 30 second alarm goes off.
It also changes the test to decide whether to enter
poll()after blocking signals to test exactly those values that might be changed by a signal. Testing these is important, testing anything else is pointless.Signed-off-by NeilBrown <[email protected]>Signed-off-by David Howells <[email protected]>
Quoted 62 lines
---
cachefilesd.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)cachefilesd.c +6
-4@@ -99,6 +99,7 @@ static struct object **cullready;static unsigned nr_in_build_table;static unsigned nr_in_ready_table;static int ncullable; +static bool cull_delayed;static const char *configfile ="/etc/cachefilesd.conf" ;@@ -246,6 +247,7 @@ static void sigio(int sig)static void sigalrm (int sig) { jumpstart_scan = true; + cull_delayed = false; }/*****************************************************************************/ @@ -613,11 +615,11 @@ static void cachefilesd(void)/* sleep without racing on reap and cull with the signal * handlers */ -if (!scan && !reap && !cull) { +if (!scan && !reap && !(cull && !cull_delayed)) {if (sigprocmask (SIG_BLOCK , &sigs, &osigs) <0 )oserror ("Unable to block signals" ); -if (!reap && !cull) { +if (!reap && !stop && !jumpstart_scan) {if (ppoll (pollfds,1 ,NULL , &osigs) <0 && errno !=EINTR )oserror ("Unable to suspend process" );@@ -644,7 +646,7 @@ static void cachefilesd(void)if (cull) {if (nr_in_ready_table >0 )cull_objects (); -else if (nr_in_build_table ==0 ) +else if (nr_in_build_table ==0 && !cull_delayed) jumpstart_scan = true; }@@ -1364,6 +1366,7 @@ static void decant_cull_table(void)/* if nothing there, scan again in a short while */ if (nr_in_build_table ==0 ) { + cull_delayed = true;signal (SIGALRM , sigalrm);alarm (30 );return ; --Linux-cachefs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/linux-cachefs
Signature · 1 line
�js
David Howells
maintainerPATCH 2/2 Suspend/resume culling based on recently released file/block counts
In reply to David HowellsNewer kernels include the counts of objects and blocks unpinned since last the status was read over the control device fd. The unpinning is usually through relinquishment by the netfs.
If these fields are present, extract them and use them to work out whether there can be anything to cull. The fields are reset when read and POLLIN is signalled when the file counter is increased from 0.
If there is nothing new to cull, scanning to build a cull table is suspended until more stuff is available. If the new fields are present, work on that basis - otherwise we sleep for 30s and try again.
The thresholds at which culling should be resumed can be set by adding:
resume_thresholds <block> <file>
to the config file. The cull scan is retried when sufficient cumulative unpinning has happened that either threshold is exceeded. The block threshold watches the cumulative i_blocks for unpinned cache inodes and the file threshold watches the number of cache inodes unpinned. If either is "-" then that threshold is disabled. The default is 5 files and no block threshold.
Further:
(*) Some of the variable names are changed to be more obvious as to what
they do.
(*) If time-based culling is used, then SIGALRM must be blocked between
the did-it-happen check and the call to poll. SIGINT, SIGTERM and SIGIO are already so blocked.
(*) All the decision making logic is moved into the cachefilesd() function
rather than being spread out.
---
README | 13 ++ cachefilesd.c | 275 +++++++++++++++++++++++++++++++++++++--------------- cachefilesd.conf.5 | 10 ++ 3 files changed, 219 insertions(+), 79 deletions(-)
README +13 -0
@@ -114,6 +114,19 @@ set up cache ready for use. The following script commands are available: entries. The permissible values are between 12 and 20, the latter indicating 1048576 entries. The default is 12. + (*) resume_thresholds <blocks> <files> + + Scanning to refill the cull table is suspended when all the objects in + a cache are pinned by a live network filesystem in the kernel and + there's nothing available to cull. Scanning is resumed when the kernel + releases sufficient objects that either the number of objects released + exceeds the files parameter here or the cumulative i_blocks values + exceed the blocks parameter. Either threshold can be disabled by + specifying it as "-". + + The default is to ignore the block threshold and to resume when five or + more files have been released. + (*) debug <mask> Specify a numeric bitmask to control debugging in the kernel module.
cachefilesd.c +196 -79
460 lines not on this page · raw message
cachefilesd.conf.5 +10 -0
@@ -75,6 +75,16 @@ amount of a systems resources, which may be undesirable. Supplying this option disables all culling activity. The cache will keep building up to the limits set and won't be shrunk, except by the removal of out-dated cache files. .TP +.B resume_thresholds <blocks> <files> +This command specifies the amount of blocks or files that the kernel should let +go of before the daemon should resume from culling table scan suspension. +.IP +Scanning to refill the cull table is suspended when all the objects in a cache +are pinned by a live network filesystem in the kernel and there's nothing to +cull. +.IP +Either value can be "-" to indicate that this threshold should be ignored. +.TP .B debug <mask> This command specifies a numeric bitmask to control debugging in the kernel module. The default is zero (all off). The following values can be OR'd into
Re: [Linux-cachefs] [PATCH 2/2] Suspend/resume culling based on recently released file/block counts
Quoted 28 lines · David Howells
Newer kernels include the counts of objects and blocks unpinned since last the status was read over the control device fd. The unpinning is usually through relinquishment by the netfs.
If these fields are present, extract them and use them to work out whether there can be anything to cull. The fields are reset when read and POLLIN is signalled when the file counter is increased from 0.
If there is nothing new to cull, scanning to build a cull table is suspended until more stuff is available. If the new fields are present, work on that basis - otherwise we sleep for 30s and try again.
The thresholds at which culling should be resumed can be set by adding:
resume_thresholds<block> <file>to the config file. The cull scan is retried when sufficient cumulative unpinning has happened that either threshold is exceeded. The block threshold watches the cumulative
i_blocksfor unpinned cache inodes and the file threshold watches the number of cache inodes unpinned. If either is "-" then that threshold is disabled. The default is 5 files and no block threshold.Further:
(*) Some of the variable names are changed to be more obvious as to what
they do.
Quoted 8 lines
(*) If time-based culling is used, then SIGALRM must be blocked between
the did-it-happen check and the call to poll. SIGINT, SIGTERM and SIGIO are already so blocked.(*) All the decision making logic is moved into the
cachefilesd()function
rather than being spread out.Signed-off-by David Howells <[email protected]>
steved.
Quoted 542 lines
---
README | 13 ++ cachefilesd.c | 275 +++++++++++++++++++++++++++++++++++++--------------- cachefilesd.conf.5 | 10 ++ 3 files changed, 219 insertions(+), 79 deletions(-)README +13
-0@@ -114,6 +114,19 @@ set up cache ready for use. The following script commands are available: entries. The permissible values are between 12 and 20, the latter indicating 1048576 entries. The default is 12. + (*) resume_thresholds <blocks> <files> + + Scanning to refill the cull table is suspended when all the objects in + a cache are pinned by a live network filesystem in the kernel and + there's nothing available to cull. Scanning is resumed when the kernel + releases sufficient objects that either the number of objects released + exceeds the files parameter here or the cumulative i_blocks values + exceed the blocks parameter. Either threshold can be disabled by + specifying it as "-". + + The default is to ignore the block threshold and to resume when five or + more files have been released. + (*) debug <mask> Specify a numeric bitmask to control debugging in the kernel module.cachefilesd.c +196
-79@@ -46,6 +46,7 @@#include <dirent.h> #include <time.h> #include <poll.h> +#include <limits.h> #include <sys/inotify.h> #include <sys/time.h> #include <sys/vfs.h> @@ -85,8 +86,8 @@ static int nobjects = 1;static int nopendir;/* current scan point */ -static struct object *scan = &root; -static bool jumpstart_scan; +static struct object *scan_cursor; +static bool scan_signalled, stop_signalled, reap_signalled;/* ranked order of cullable objects * - we have two tables: one we're building and one that's full of ready to be @@ -99,7 +100,10 @@ static struct object **cullready;static unsigned nr_in_build_table;static unsigned nr_in_ready_table;static int ncullable; -static bool cull_delayed; +static bool kernel_wants_cull; +static bool have_nr_releases; +static unsigned long long f_released_since_last_scan; +static unsigned long long b_released_since_last_scan;static const char *configfile ="/etc/cachefilesd.conf" ;@@ -108,11 +112,13 @@ static const char *procfile = "/proc/fs/cachefiles";static const char *pidfile ="/var/run/cachefilesd.pid" ;static char *cacheroot, *graveyardpath; +static bool culling_disabled;static bool xnolog, xopenedlog;static int xdebug; -static bool nocull, stop, reap, cull;static int graveyardfd;static unsigned long long brun, bcull, bstop, frun, fcull, fstop; +static unsigned long long b_resume_threshold =ULLONG_MAX ; +static unsigned long long f_resume_threshold =5 ;#define cachefd 3@@ -212,7 +218,8 @@ static void reap_graveyard_aux(const char *dirname);static void read_cache_state (void );static int is_object_in_use (const char *filename);static void cull_file (const char *filename); -static void build_cull_table (void ); +static void begin_building_cull_table (void ); +static bool build_cull_table (void );static void decant_cull_table (void );static void insert_into_cull_table (struct object *object);static void put_object (struct object *object);@@ -228,7 +235,7 @@ static void cull_objects(void);*/ static void sigterm (int sig) { - stop = true; + stop_signalled = true; }/*****************************************************************************/ @@ -237,7 +244,7 @@ static void sigterm(int sig)*/ static void sigio (int sig) { - reap = true; + reap_signalled = true; }/*****************************************************************************/ @@ -246,8 +253,7 @@ static void sigio(int sig)*/ static void sigalrm (int sig) { - jumpstart_scan = true; - cull_delayed = false; + scan_signalled = true; }/*****************************************************************************/ @@ -312,7 +318,7 @@ int main(int argc, char *argv[])case 'N' :/* disable culling */ - nocull = true; + culling_disabled = true;break ;case 'f' :@@ -420,7 +426,7 @@ int main(int argc, char *argv[])/* allow culling to be disabled */ if (memcmp (cp,"nocull" ,6 ) ==0 && (!cp[6 ] ||isspace (cp[6 ]))) { - nocull = true; + culling_disabled = true; }/* note the cull table size command */ @@ -439,6 +445,40 @@ int main(int argc, char *argv[])continue ; } +/* Note the suspension resume released file count thresholds +* ("-" to disable a threshold). +*/ +if (memcmp (cp,"resume_thresholds" ,18 ) ==0 &&isspace (cp[18 ])) { +unsigned long long b_thresh, f_thresh; +char *sp; + +for (sp = cp +18 ;isspace (*sp); sp++) {;} + +if (*sp =='-' ) { + sp++; + b_thresh =ULLONG_MAX ; + }else { + b_thresh =strtoul (sp, &sp,10 ); + } + +if (!*sp || !isspace (*sp) || b_thresh ==0 ) +cfgerror ("Invalid resume threshold (blocks)" ); +for (;isspace (*sp); sp++) {;} + +if (*sp =='-' ) { + sp++; + f_thresh =ULLONG_MAX ; + }else { + f_thresh =strtoul (sp, &sp,10 ); +if (*sp || f_thresh ==0 ) +cfgerror ("Invalid resume threshold (files)" ); + } + + b_resume_threshold = b_thresh; + f_resume_threshold = f_thresh; +continue ; + } +/* note the dir command */ if (memcmp (cp,"dir" ,3 ) ==0 &&isspace (cp[3 ])) {char *sp;@@ -479,7 +519,7 @@ int main(int argc, char *argv[])oserror ("Unable to close %s" , configfile);/* allocate the cull tables */ -if (!nocull) { +if (!culling_disabled) { cullbuild =calloc (culltable_size,sizeof (cullbuild[0 ]));if (!cullbuild)oserror ("calloc" );@@ -583,6 +623,8 @@ static void open_cache(void)static void cachefilesd (void ) {sigset_t sigs, osigs; +bool scanning_suspended = false; +bool scan_in_progress = false;struct pollfd pollfds[1 ] = { [0 ] = {@@ -596,13 +638,14 @@ static void cachefilesd(void)/* open the cache directories */ open_cache (); -/* we need to disable I/O and termination signals so they're only -* caught at appropriate times +/* We need to be able to disable signals that we need to check for +* before calling poll so that we don't race and miss something. */ sigemptyset (&sigs);sigaddset (&sigs,SIGIO );sigaddset (&sigs,SIGINT );sigaddset (&sigs,SIGTERM ); +sigaddset (&sigs,SIGALRM );signal (SIGTERM , sigterm);signal (SIGINT , sigterm);@@ -610,16 +653,81 @@ static void cachefilesd(void)/* check the graveyard for graves */ reap_graveyard (); -while (!stop) { +while (!stop_signalled) { +bool do_cull = false; + +debug (3 ,"Loop %sbuild=%d ready=%d susp=%u scan=%u" , + culling_disabled ?"NOCULL " :"" , + nr_in_build_table, nr_in_ready_table, + scanning_suspended, scan_in_progress); +read_cache_state (); -/* sleep without racing on reap and cull with the signal -* handlers */ -if (!scan && !reap && !(cull && !cull_delayed)) { +if (!culling_disabled) { +/* Determine if we're going to need to start a new scan +* to refill the cull table. We want to do this if the +* secondary cull table is less than half full - but +* overriding that, we don't want to do this if we know +* there's insufficient cullables to make it worth +* while. +*/ +if (!scan_in_progress) { +bool begin_scan = false; + +debug (1 ,"Consider scan %d/%d" , + nr_in_build_table, culltable_size /2 ); + +if (nr_in_build_table < culltable_size /2 ) { +debug (1 ,"Want to scan" ); + begin_scan = true; + } + +if (begin_scan && scanning_suspended) { +debug (1 ,"Scanning suspended" ); +if (have_nr_releases) { +if (f_released_since_last_scan < + f_resume_threshold && + b_released_since_last_scan < + b_resume_threshold) + begin_scan = false; + }else { + begin_scan = scan_signalled; + } + } + +if (begin_scan) { +debug (1 ,"Beginning a scan" ); +begin_building_cull_table (); + scan_in_progress = true; + scanning_suspended = false; + scan_signalled = false; + f_released_since_last_scan =0 ; + b_released_since_last_scan =0 ; + } + } + +/* Determine if there's anything we can actually cull yet if +* the kernel is calling for space. +*/ +if (kernel_wants_cull) { +debug (1 ,"Want to cull" ); +if (nr_in_ready_table >0 ) + do_cull = true; + } + } + +/* We block the signals across the checks for reap, cull and +* scan initiation before polling so that we sleep without +* racing against the signal handlers. +*/ +if (!scan_in_progress && !reap_signalled && !do_cull) {if (sigprocmask (SIG_BLOCK , &sigs, &osigs) <0 )oserror ("Unable to block signals" ); -if (!reap && !stop && !jumpstart_scan) { +if (!reap_signalled && + !stop_signalled && + !scan_signalled) { +debug (1 ,"Poll" );if (ppoll (pollfds,1 ,NULL , &osigs) <0 && errno !=EINTR )oserror ("Unable to suspend process" );@@ -627,37 +735,44 @@ static void cachefilesd(void)if (sigprocmask (SIG_UNBLOCK , &sigs,NULL ) <0 )oserror ("Unable to unblock signals" ); - -read_cache_state (); +continue ; } -if (nocull) { - cull = false; - }else { -if (jumpstart_scan) { - jumpstart_scan = false; -if (!stop && !scan) { -debug (1 ,"Refilling cull table" ); - root.usage++; - scan = &root; +if (!culling_disabled) { +if (do_cull) +cull_objects (); + +if (scan_in_progress) { + scan_in_progress =build_cull_table (); +if (!scan_in_progress) { +/* Scan complete. +* +* If the scan didn't produce a full +* table then don't repeat the scan +* until something gets released by the +* kernel. +*/ +if (nr_in_build_table < culltable_size) { +debug (1 ,"Suspend scanning" ); + scanning_suspended = true; +if (!have_nr_releases) { +signal (SIGALRM , sigalrm); +alarm (30 ); + } + } } } -if (cull) { -if (nr_in_ready_table >0 ) -cull_objects (); -else if (nr_in_build_table ==0 && !cull_delayed) - jumpstart_scan = true; +if (!scan_in_progress) { +if (nr_in_ready_table <= culltable_size /2 +2 && + nr_in_build_table >0 ) { +debug (1 ,"Decant" ); +decant_cull_table (); + } } - -if (scan) -build_cull_table (); - -if (!scan && nr_in_ready_table ==0 && nr_in_build_table >0 ) -decant_cull_table (); } -if (reap) +if (reap_signalled)reap_graveyard (); }@@ -672,7 +787,7 @@ static void cachefilesd(void)static void reap_graveyard (void ) {/* set a one-shot notification to catch more graves appearing */ - reap = false; + reap_signalled = false;signal (SIGIO , sigio);if (fcntl (graveyardfd,F_NOTIFY ,DN_CREATE ) <0 )oserror ("unable to set notification on graveyard" );@@ -763,6 +878,8 @@ static void read_cache_state(void)oserror ("Unable to read cache state" ); buffer[n] ='\0' ; +debug (3 ,"KERNEL: %s" , buffer); + tok = buffer; do { next = strpbrk(tok, " \t");@@ -777,20 +894,27 @@ static void read_cache_state(void)continue ; } -if (strcmp (tok,"cull" ) ==0 ) - cull = (strtoul (arg,NULL ,0 ) !=0 ); -else if (strcmp (tok,"brun" ) ==0 ) +if (strcmp (tok,"cull" ) ==0 ) { + kernel_wants_cull = (strtoul (arg,NULL ,0 ) !=0 ); + }else if (strcmp (tok,"brun" ) ==0 ) { brun =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"bcull" ) ==0 ) + }else if (strcmp (tok,"bcull" ) ==0 ) { bcull =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"bstop" ) ==0 ) + }else if (strcmp (tok,"bstop" ) ==0 ) { bstop =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"frun" ) ==0 ) + }else if (strcmp (tok,"frun" ) ==0 ) { frun =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"fcull" ) ==0 ) + }else if (strcmp (tok,"fcull" ) ==0 ) { fcull =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"fstop" ) ==0 ) + }else if (strcmp (tok,"fstop" ) ==0 ) { fstop =strtoull (arg,NULL ,16 ); + }else if (strcmp (tok,"breleased" ) ==0 ) { + b_released_since_last_scan +=strtoull (arg,NULL ,16 ); + have_nr_releases = true; + }else if (strcmp (tok,"freleased" ) ==0 ) { + f_released_since_last_scan +=strtoull (arg,NULL ,16 ); + have_nr_releases = true; + } }while ((tok = next)); }@@ -1123,9 +1247,21 @@ static void insert_into_cull_table(struct object *object)/*****************************************************************************/ /* -* do the next step in building up the cull table +* Begin a scan to build a cull table. */ -static void build_cull_table (void ) +static void begin_building_cull_table (void ) +{ +debug (1 ,"Refilling cull table" ); + root.usage++; + scan_cursor = &root; +} + +/*****************************************************************************/ +/* +* Do the next step in building up the cull table. Returns false upon +* completion of a scan. +*/ +static bool build_cull_table (void ) {struct dirent dirent, *de;struct object *curr, *child;@@ -1133,7 +1269,7 @@ static void build_cull_table(void)unsigned loop;int fd; - curr = scan; + curr = scan_cursor;if (!curr->dir) { curr->empty = true;@@ -1289,10 +1425,10 @@ next:debug (2 ,"- descend" ); child->new = false; - scan = child; + scan_cursor = child;debug (2 ,"<-- build_cull_table({%s})" , curr->name); -return ; +return true;default :error ("Unexpected type" );@@ -1335,15 +1471,13 @@ dir_read_complete: } } - scan = curr->parent; -if (!scan) { + scan_cursor = curr->parent; +if (!scan_cursor)debug (1 ,"Scan complete" ); -decant_cull_table (); - }debug (2 ,"<-- build_cull_table({%s})" , curr->name);put_object (curr); -return ; +return scan_cursor !=NULL ;/* delete unexpected objects that we've found */ found_unexpected_object:@@ -1361,17 +1495,9 @@ static void decant_cull_table(void) {unsigned loop, avail, copy, leave, space, n; -if (scan) +if (scan_cursor)error ("Can't decant cull table whilst scanning" ); -/* if nothing there, scan again in a short while */ -if (nr_in_build_table ==0 ) { - cull_delayed = true; -signal (SIGALRM , sigalrm); -alarm (30 ); -return ; - } -/* mark the new entries cullable */ for (loop =0 ; loop < nr_in_build_table; loop++) {if (!cullbuild[loop]->cullable) {@@ -1504,13 +1630,4 @@ static void cull_objects(void)cull_object (cullready[nr_in_ready_table -1 ]); cullready[--nr_in_ready_table] = (void *)(0x6b000000 | __LINE__); } - -/* must start refilling the cull table */ -if (!scan && nr_in_build_table < culltable_size /2 +2 ) { -decant_cull_table (); - -debug (1 ,"Refilling cull table" ); - root.usage++; - scan = &root; - } }cachefilesd.conf.5 +10
-0@@ -75,6 +75,16 @@ amount of a systems resources, which may be undesirable. Supplying this option disables all culling activity. The cache will keep building up to the limits set and won't be shrunk, except by the removal of out-dated cache files. .TP +.B resume_thresholds <blocks> <files> +This command specifies the amount of blocks or files that the kernel should let +go of before the daemon should resume from culling table scan suspension. +.IP +Scanning to refill the cull table is suspended when all the objects in a cache +are pinned by a live network filesystem in the kernel and there's nothing to +cull. +.IP +Either value can be "-" to indicate that this threshold should be ignored. +.TP .B debug <mask> This command specifies a numeric bitmask to control debugging in the kernel module. The default is zero (all off). The following values can be OR'd into
Quoted 19 lines · David Howells
Provide read-and-reset objects- and blocks-released counters for cachefilesd to use to work out whether there's anything new that can be culled.
One of the problems cachefilesd has is that if all the objects in the cache are pinned by inodes lying dormant in the kernel inode cache, there isn't anything for it to cull. In such a case, it just spins around walking the filesystem tree and scanning for something to cull. This eats up a lot of CPU time.
By telling cachefilesd if there have been any releases, the daemon can sleep until there is the possibility of something to do.
cachefilesd finds this information by the following means:
(1) When the control fd is read, the kernel presents a list of values of interest. "freleased=N" and "breleased=N" are added to this list to indicate the number of files released and number of blocks released since the last read call. At this point the counters are reset.
Quoted 8 lines
(2) POLLIN is signalled if the number of files released becomes greater
than 0.Note that by 'released' it just means that the kernel has released its interest in those files for the moment, not necessarily that the files should be deleted from the cache.
Signed-off-by David Howells <[email protected]>
steved.
Quoted 143 lines
---
fs/cachefiles/daemon.c | 13 ++++++++++--- fs/cachefiles/interface.c | 11 ++--------- fs/cachefiles/internal.h | 4 ++++ fs/cachefiles/namei.c | 28 +++++++++++++++++++++++----- 4 files changed, 39 insertions(+), 17 deletions(-)fs/cachefiles/daemon.c +10
-3@@ -162,6 +162,8 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, size_t buflen, loff_t *pos) { struct cachefiles_cache *cache = file->private_data; +unsigned long long b_released; +unsigned f_released;char buffer[256 ];int n;@@ -174,6 +176,8 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer,cachefiles_has_space (cache,0 ,0 );/* summarise */ + f_released =atomic_xchg (&cache->f_released,0 ); + b_released =atomic_long_xchg (&cache->b_released,0 );clear_bit (CACHEFILES_STATE_CHANGED , &cache->flags); n = snprintf(buffer, sizeof(buffer),@@ -183,15 +187,18 @@ static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer," fstop=%llx" " brun=%llx" " bcull=%llx" -" bstop=%llx" , +" bstop=%llx" +" freleased=%x" +" breleased=%llx" ,test_bit (CACHEFILES_CULLING , &cache->flags) ?'1' :'0' , (unsigned long long ) cache->frun, (unsigned long long ) cache->fcull, (unsigned long long ) cache->fstop, (unsigned long long ) cache->brun, (unsigned long long ) cache->bcull, - (unsigned long long ) cache->bstop - ); + (unsigned long long ) cache->bstop, + f_released, + b_released);if (n > buflen)return -EMSGSIZE ;fs/cachefiles/interface.c +2
-9@@ -291,15 +291,8 @@ static void cachefiles_drop_object(struct fscache_object *_object) }/* note that the object is now inactive */ -if (test_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags)) { -write_lock (&cache->active_lock); -if (!test_and_clear_bit (CACHEFILES_OBJECT_ACTIVE , - &object->flags)) -BUG (); -rb_erase (&object->active_node, &cache->active_nodes); -wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); -write_unlock (&cache->active_lock); - } +if (test_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags)) +cachefiles_mark_object_inactive (cache, object);dput (object->dentry); object->dentry =NULL ;fs/cachefiles/internal.h +4
-0@@ -66,6 +66,8 @@ struct cachefiles_cache {struct rb_root active_nodes;/* active nodes (can't be culled) */ rwlock_t active_lock;/* lock for active_nodes */ atomic_t gravecounter;/* graveyard uniquifier */ +atomic_t f_released;/* number of objects released lately */ +atomic_long_t b_released;/* number of blocks released lately */ unsigned frun_percent;/* when to stop culling (% files) */ unsigned fcull_percent;/* when to start culling (% files) */ unsigned fstop_percent;/* when to stop allocating (% files) */ @@ -157,6 +159,8 @@ extern char *cachefiles_cook_key(const u8 *raw, int keylen, uint8_t type);/* * namei.c */ +extern void cachefiles_mark_object_inactive (struct cachefiles_cache *cache, +struct cachefiles_object *object);extern int cachefiles_delete_object (struct cachefiles_cache *cache,struct cachefiles_object *object); extern int cachefiles_walk_to_object(struct cachefiles_object *parent,fs/cachefiles/namei.c +23
-5@@ -258,6 +258,28 @@ requeue: }/* +* Mark an object as being inactive. +*/ +void cachefiles_mark_object_inactive (struct cachefiles_cache *cache, +struct cachefiles_object *object) +{ +write_lock (&cache->active_lock); +rb_erase (&object->active_node, &cache->active_nodes); +clear_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags); +write_unlock (&cache->active_lock); + +wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); + +/* This object can now be culled, so we need to let the daemon know +* that there is something it can remove if it needs to. +*/ +atomic_long_add (d_backing_inode (object->dentry)->i_blocks, + &cache->b_released); +if (atomic_inc_return (&cache->f_released)) +cachefiles_state_changed (cache); +} + +/* * delete an object representation from the cache * - file backed objects are unlinked * - directory backed objects are stuffed into the graveyard for userspace to @@ -684,11 +706,7 @@ mark_active_timed_out: check_error:_debug ("check error %d" , ret); -write_lock (&cache->active_lock); -rb_erase (&object->active_node, &cache->active_nodes); -clear_bit (CACHEFILES_OBJECT_ACTIVE , &object->flags); -wake_up_bit (&object->flags,CACHEFILES_OBJECT_ACTIVE ); -write_unlock (&cache->active_lock); +cachefiles_mark_object_inactive (cache, object); release_dentry:dput (object->dentry); object->dentry =NULL ;
John Snow
Reviewed-byRe: [Linux-cachefs] [PATCH 2/2] Suspend/resume culling based on recently released file/block counts
Quoted 185 lines · David Howells
Newer kernels include the counts of objects and blocks unpinned since last the status was read over the control device fd. The unpinning is usually through relinquishment by the netfs.
If these fields are present, extract them and use them to work out whether there can be anything to cull. The fields are reset when read and POLLIN is signalled when the file counter is increased from 0.
If there is nothing new to cull, scanning to build a cull table is suspended until more stuff is available. If the new fields are present, work on that basis - otherwise we sleep for 30s and try again.
The thresholds at which culling should be resumed can be set by adding:
resume_thresholds<block> <file>to the config file. The cull scan is retried when sufficient cumulative unpinning has happened that either threshold is exceeded. The block threshold watches the cumulative
i_blocksfor unpinned cache inodes and the file threshold watches the number of cache inodes unpinned. If either is "-" then that threshold is disabled. The default is 5 files and no block threshold.Further:
(*) Some of the variable names are changed to be more obvious as to what
they do.(*) If time-based culling is used, then SIGALRM must be blocked between
the did-it-happen check and the call to poll. SIGINT, SIGTERM and SIGIO are already so blocked.(*) All the decision making logic is moved into the
cachefilesd()function
rather than being spread out.Signed-off-by David Howells <[email protected]>---
README | 13 ++ cachefilesd.c | 275 +++++++++++++++++++++++++++++++++++++--------------- cachefilesd.conf.5 | 10 ++ 3 files changed, 219 insertions(+), 79 deletions(-)README +13
-0@@ -114,6 +114,19 @@ set up cache ready for use. The following script commands are available: entries. The permissible values are between 12 and 20, the latter indicating 1048576 entries. The default is 12. + (*) resume_thresholds <blocks> <files> + + Scanning to refill the cull table is suspended when all the objects in + a cache are pinned by a live network filesystem in the kernel and + there's nothing available to cull. Scanning is resumed when the kernel + releases sufficient objects that either the number of objects released + exceeds the files parameter here or the cumulative i_blocks values + exceed the blocks parameter. Either threshold can be disabled by + specifying it as "-". + + The default is to ignore the block threshold and to resume when five or + more files have been released. + (*) debug <mask> Specify a numeric bitmask to control debugging in the kernel module.cachefilesd.c +25
-11@@ -46,6 +46,7 @@#include <dirent.h> #include <time.h> #include <poll.h> +#include <limits.h> #include <sys/inotify.h> #include <sys/time.h> #include <sys/vfs.h> @@ -85,8 +86,8 @@ static int nobjects = 1;static int nopendir;/* current scan point */ -static struct object *scan = &root; -static bool jumpstart_scan; +static struct object *scan_cursor; +static bool scan_signalled, stop_signalled, reap_signalled;/* ranked order of cullable objects * - we have two tables: one we're building and one that's full of ready to be @@ -99,7 +100,10 @@ static struct object **cullready;static unsigned nr_in_build_table;static unsigned nr_in_ready_table;static int ncullable; -static bool cull_delayed; +static bool kernel_wants_cull; +static bool have_nr_releases; +static unsigned long long f_released_since_last_scan; +static unsigned long long b_released_since_last_scan;static const char *configfile ="/etc/cachefilesd.conf" ;@@ -108,11 +112,13 @@ static const char *procfile = "/proc/fs/cachefiles";static const char *pidfile ="/var/run/cachefilesd.pid" ;static char *cacheroot, *graveyardpath; +static bool culling_disabled;static bool xnolog, xopenedlog;static int xdebug; -static bool nocull, stop, reap, cull;static int graveyardfd;static unsigned long long brun, bcull, bstop, frun, fcull, fstop; +static unsigned long long b_resume_threshold =ULLONG_MAX ; +static unsigned long long f_resume_threshold =5 ;#define cachefd 3@@ -212,7 +218,8 @@ static void reap_graveyard_aux(const char *dirname);static void read_cache_state (void );static int is_object_in_use (const char *filename);static void cull_file (const char *filename); -static void build_cull_table (void ); +static void begin_building_cull_table (void ); +static bool build_cull_table (void );static void decant_cull_table (void );static void insert_into_cull_table (struct object *object);static void put_object (struct object *object);@@ -228,7 +235,7 @@ static void cull_objects(void);*/ static void sigterm (int sig) { - stop = true; + stop_signalled = true; }/*****************************************************************************/ @@ -237,7 +244,7 @@ static void sigterm(int sig)*/ static void sigio (int sig) { - reap = true; + reap_signalled = true; }/*****************************************************************************/ @@ -246,8 +253,7 @@ static void sigio(int sig)*/ static void sigalrm (int sig) { - jumpstart_scan = true; - cull_delayed = false; + scan_signalled = true; }/*****************************************************************************/ @@ -312,7 +318,7 @@ int main(int argc, char *argv[])case 'N' :/* disable culling */ - nocull = true; + culling_disabled = true;break ;case 'f' :@@ -420,7 +426,7 @@ int main(int argc, char *argv[])/* allow culling to be disabled */ if (memcmp (cp,"nocull" ,6 ) ==0 && (!cp[6 ] ||isspace (cp[6 ]))) { - nocull = true; + culling_disabled = true; }/* note the cull table size command */ @@ -439,6 +445,40 @@ int main(int argc, char *argv[])continue ; } +/* Note the suspension resume released file count thresholds +* ("-" to disable a threshold). +*/ +if (memcmp (cp,"resume_thresholds" ,18 ) ==0 &&isspace (cp[18 ])) { +unsigned long long b_thresh, f_thresh; +char *sp; + +for (sp = cp +18 ;isspace (*sp); sp++) {;}
cachefilesd.c int main(int argc, char *argv[]) · line 465
Quoted 10 lines
cachefilesd.c +10
-0+ +if (*sp =='-' ) { + sp++; + b_thresh =ULLONG_MAX ; + }else { + b_thresh =strtoul (sp, &sp,10 ); + } + +if (!*sp || !isspace (*sp) || b_thresh ==0 ) +cfgerror ("Invalid resume threshold (blocks)" );
Seems to me like we're mixing parsing errors with invalid configuration error messages, but I'm just bike-shedding aboard the SS Howells.
Quoted 57 lines
cachefilesd.c +22
-3+for (;isspace (*sp); sp++) {;} + +if (*sp =='-' ) { + sp++; + f_thresh =ULLONG_MAX ; + }else { + f_thresh =strtoul (sp, &sp,10 ); +if (*sp || f_thresh ==0 ) +cfgerror ("Invalid resume threshold (files)" ); + } + + b_resume_threshold = b_thresh; + f_resume_threshold = f_thresh; +continue ; + } +/* note the dir command */ if (memcmp (cp,"dir" ,3 ) ==0 &&isspace (cp[3 ])) {char *sp;@@ -479,7 +519,7 @@ int main(int argc, char *argv[])oserror ("Unable to close %s" , configfile);/* allocate the cull tables */ -if (!nocull) { +if (!culling_disabled) { cullbuild =calloc (culltable_size,sizeof (cullbuild[0 ]));if (!cullbuild)oserror ("calloc" );@@ -583,6 +623,8 @@ static void open_cache(void)static void cachefilesd (void ) {sigset_t sigs, osigs; +bool scanning_suspended = false; +bool scan_in_progress = false;struct pollfd pollfds[1 ] = { [0 ] = {@@ -596,13 +638,14 @@ static void cachefilesd(void)/* open the cache directories */ open_cache (); -/* we need to disable I/O and termination signals so they're only -* caught at appropriate times +/* We need to be able to disable signals that we need to check for +* before calling poll so that we don't race and miss something. */ sigemptyset (&sigs);sigaddset (&sigs,SIGIO );sigaddset (&sigs,SIGINT );sigaddset (&sigs,SIGTERM ); +sigaddset (&sigs,SIGALRM );signal (SIGTERM , sigterm);signal (SIGINT , sigterm);@@ -610,16 +653,81 @@ static void cachefilesd(void)/* check the graveyard for graves */ reap_graveyard ();
cachefilesd.c static void cachefilesd(void) · line 663
Quoted 10 lines
cachefilesd.c +8
-1-while (!stop) { +while (!stop_signalled) { +bool do_cull = false; + +debug (3 ,"Loop %sbuild=%d ready=%d susp=%u scan=%u" , + culling_disabled ?"NOCULL " :"" , + nr_in_build_table, nr_in_ready_table, + scanning_suspended, scan_in_progress); +
Trailing whitespace
Quoted 152 lines
cachefilesd.c +93
-28read_cache_state (); -/* sleep without racing on reap and cull with the signal -* handlers */ -if (!scan && !reap && !(cull && !cull_delayed)) { +if (!culling_disabled) { +/* Determine if we're going to need to start a new scan +* to refill the cull table. We want to do this if the +* secondary cull table is less than half full - but +* overriding that, we don't want to do this if we know +* there's insufficient cullables to make it worth +* while. +*/ +if (!scan_in_progress) { +bool begin_scan = false; + +debug (1 ,"Consider scan %d/%d" , + nr_in_build_table, culltable_size /2 ); + +if (nr_in_build_table < culltable_size /2 ) { +debug (1 ,"Want to scan" ); + begin_scan = true; + } + +if (begin_scan && scanning_suspended) { +debug (1 ,"Scanning suspended" ); +if (have_nr_releases) { +if (f_released_since_last_scan < + f_resume_threshold && + b_released_since_last_scan < + b_resume_threshold) + begin_scan = false; + }else { + begin_scan = scan_signalled; + } + } + +if (begin_scan) { +debug (1 ,"Beginning a scan" ); +begin_building_cull_table (); + scan_in_progress = true; + scanning_suspended = false; + scan_signalled = false; + f_released_since_last_scan =0 ; + b_released_since_last_scan =0 ; + } + } + +/* Determine if there's anything we can actually cull yet if +* the kernel is calling for space. +*/ +if (kernel_wants_cull) { +debug (1 ,"Want to cull" ); +if (nr_in_ready_table >0 ) + do_cull = true; + } + } + +/* We block the signals across the checks for reap, cull and +* scan initiation before polling so that we sleep without +* racing against the signal handlers. +*/ +if (!scan_in_progress && !reap_signalled && !do_cull) {if (sigprocmask (SIG_BLOCK , &sigs, &osigs) <0 )oserror ("Unable to block signals" ); -if (!reap && !stop && !jumpstart_scan) { +if (!reap_signalled && + !stop_signalled && + !scan_signalled) { +debug (1 ,"Poll" );if (ppoll (pollfds,1 ,NULL , &osigs) <0 && errno !=EINTR )oserror ("Unable to suspend process" );@@ -627,37 +735,44 @@ static void cachefilesd(void)if (sigprocmask (SIG_UNBLOCK , &sigs,NULL ) <0 )oserror ("Unable to unblock signals" ); - -read_cache_state (); +continue ; } -if (nocull) { - cull = false; - }else { -if (jumpstart_scan) { - jumpstart_scan = false; -if (!stop && !scan) { -debug (1 ,"Refilling cull table" ); - root.usage++; - scan = &root; +if (!culling_disabled) { +if (do_cull) +cull_objects (); + +if (scan_in_progress) { + scan_in_progress =build_cull_table (); +if (!scan_in_progress) { +/* Scan complete. +* +* If the scan didn't produce a full +* table then don't repeat the scan +* until something gets released by the +* kernel. +*/ +if (nr_in_build_table < culltable_size) { +debug (1 ,"Suspend scanning" ); + scanning_suspended = true; +if (!have_nr_releases) { +signal (SIGALRM , sigalrm); +alarm (30 ); + } + } } } -if (cull) { -if (nr_in_ready_table >0 ) -cull_objects (); -else if (nr_in_build_table ==0 && !cull_delayed) - jumpstart_scan = true; +if (!scan_in_progress) { +if (nr_in_ready_table <= culltable_size /2 +2 && + nr_in_build_table >0 ) { +debug (1 ,"Decant" ); +decant_cull_table (); + } } - -if (scan) -build_cull_table (); - -if (!scan && nr_in_ready_table ==0 && nr_in_build_table >0 ) -decant_cull_table (); } -if (reap) +if (reap_signalled)reap_graveyard (); }@@ -672,7 +787,7 @@ static void cachefilesd(void)static void reap_graveyard (void ) {/* set a one-shot notification to catch more graves appearing */ - reap = false; + reap_signalled = false;signal (SIGIO , sigio);if (fcntl (graveyardfd,F_NOTIFY ,DN_CREATE ) <0 )oserror ("unable to set notification on graveyard" );
cachefilesd.c static void read_cache_state(void) · line 882
Quoted 5 lines
cachefilesd.c +2
-0@@ -763,6 +878,8 @@ static void read_cache_state(void)oserror ("Unable to read cache state" ); buffer[n] ='\0' ; +debug (3 ,"KERNEL: %s" , buffer); +
Trailing whitespace
Quoted 160 lines
cachefilesd.c +36
-36tok = buffer; do { next = strpbrk(tok, " \t");@@ -777,20 +894,27 @@ static void read_cache_state(void)continue ; } -if (strcmp (tok,"cull" ) ==0 ) - cull = (strtoul (arg,NULL ,0 ) !=0 ); -else if (strcmp (tok,"brun" ) ==0 ) +if (strcmp (tok,"cull" ) ==0 ) { + kernel_wants_cull = (strtoul (arg,NULL ,0 ) !=0 ); + }else if (strcmp (tok,"brun" ) ==0 ) { brun =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"bcull" ) ==0 ) + }else if (strcmp (tok,"bcull" ) ==0 ) { bcull =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"bstop" ) ==0 ) + }else if (strcmp (tok,"bstop" ) ==0 ) { bstop =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"frun" ) ==0 ) + }else if (strcmp (tok,"frun" ) ==0 ) { frun =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"fcull" ) ==0 ) + }else if (strcmp (tok,"fcull" ) ==0 ) { fcull =strtoull (arg,NULL ,16 ); -else if (strcmp (tok,"fstop" ) ==0 ) + }else if (strcmp (tok,"fstop" ) ==0 ) { fstop =strtoull (arg,NULL ,16 ); + }else if (strcmp (tok,"breleased" ) ==0 ) { + b_released_since_last_scan +=strtoull (arg,NULL ,16 ); + have_nr_releases = true; + }else if (strcmp (tok,"freleased" ) ==0 ) { + f_released_since_last_scan +=strtoull (arg,NULL ,16 ); + have_nr_releases = true; + } }while ((tok = next)); }@@ -1123,9 +1247,21 @@ static void insert_into_cull_table(struct object *object)/*****************************************************************************/ /* -* do the next step in building up the cull table +* Begin a scan to build a cull table. */ -static void build_cull_table (void ) +static void begin_building_cull_table (void ) +{ +debug (1 ,"Refilling cull table" ); + root.usage++; + scan_cursor = &root; +} + +/*****************************************************************************/ +/* +* Do the next step in building up the cull table. Returns false upon +* completion of a scan. +*/ +static bool build_cull_table (void ) {struct dirent dirent, *de;struct object *curr, *child;@@ -1133,7 +1269,7 @@ static void build_cull_table(void)unsigned loop;int fd; - curr = scan; + curr = scan_cursor;if (!curr->dir) { curr->empty = true;@@ -1289,10 +1425,10 @@ next:debug (2 ,"- descend" ); child->new = false; - scan = child; + scan_cursor = child;debug (2 ,"<-- build_cull_table({%s})" , curr->name); -return ; +return true;default :error ("Unexpected type" );@@ -1335,15 +1471,13 @@ dir_read_complete: } } - scan = curr->parent; -if (!scan) { + scan_cursor = curr->parent; +if (!scan_cursor)debug (1 ,"Scan complete" ); -decant_cull_table (); - }debug (2 ,"<-- build_cull_table({%s})" , curr->name);put_object (curr); -return ; +return scan_cursor !=NULL ;/* delete unexpected objects that we've found */ found_unexpected_object:@@ -1361,17 +1495,9 @@ static void decant_cull_table(void) {unsigned loop, avail, copy, leave, space, n; -if (scan) +if (scan_cursor)error ("Can't decant cull table whilst scanning" ); -/* if nothing there, scan again in a short while */ -if (nr_in_build_table ==0 ) { - cull_delayed = true; -signal (SIGALRM , sigalrm); -alarm (30 ); -return ; - } -/* mark the new entries cullable */ for (loop =0 ; loop < nr_in_build_table; loop++) {if (!cullbuild[loop]->cullable) {@@ -1504,13 +1630,4 @@ static void cull_objects(void)cull_object (cullready[nr_in_ready_table -1 ]); cullready[--nr_in_ready_table] = (void *)(0x6b000000 | __LINE__); } - -/* must start refilling the cull table */ -if (!scan && nr_in_build_table < culltable_size /2 +2 ) { -decant_cull_table (); - -debug (1 ,"Refilling cull table" ); - root.usage++; - scan = &root; - } }cachefilesd.conf.5 +10
-1@@ -75,6 +75,16 @@ amount of a systems resources, which may be undesirable. Supplying this option disables all culling activity. The cache will keep building up to the limits set and won't be shrunk, except by the removal of out-dated cache files. .TP +.B resume_thresholds <blocks> <files> +This command specifies the amount of blocks or files that the kernel should let +go of before the daemon should resume from culling table scan suspension. +.IP +Scanning to refill the cull table is suspended when all the objects in a cache +are pinned by a live network filesystem in the kernel and there's nothing to +cull. +.IP +Either value can be "-" to indicate that this threshold should be ignored. +.TP .B debug <mask> This command specifies a numeric bitmask to control debugging in the kernel module. The default is zero (all off). The following values can be OR'd into --
Quoted 4 lines
Linux-cachefs mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/linux-cachefs
David Howells
maintainerRe: [Linux-cachefs] [PATCH 2/2] Suspend/resume culling based on recently released file/block counts
Quoted 5 lines · John Snow
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (!*sp || !isspace(*sp) || b_thresh == 0) + cfgerror("Invalid resume threshold (blocks)");Seems to me like we're mixing parsing errors with invalid configuration error messages, but I'm just bike-shedding aboard the SS Howells.
How about this further modification?
- if (!*sp || !isspace(*sp) || b_thresh == 0) + if (!*sp || !isspace(*sp))
+ cfgerror("Error parsing resume threshold (blocks)");
+ if (b_thresh == 0)
cfgerror("Invalid resume threshold (blocks)");
for (; isspace(*sp); sp++) {;}
@@ -470,7 +472,9 @@ int main(int argc, char *argv[])
f_thresh = ULLONG_MAX;
} else {
f_thresh = strtoul(sp, &sp, 10);
- if (*sp || f_thresh == 0)
+ if (*sp)
+ cfgerror("Error parsing resume threshold (files)");
+ if (f_thresh == 0)
cfgerror("Invalid resume threshold (files)");
David
Re: [Linux-cachefs] [PATCH 2/2] Suspend/resume culling based on recently released file/block counts
Quoted 27 lines · David Howells
Quoted 5 lines · John Snow
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (!*sp || !isspace(*sp) || b_thresh == 0) + cfgerror("Invalid resume threshold (blocks)");Seems to me like we're mixing parsing errors with invalid configuration error messages, but I'm just bike-shedding aboard the SS Howells.
How about this further modification?
(quoted diff) +6
-2- if (!*sp || !isspace(*sp) || b_thresh == 0) + if (!*sp || !isspace(*sp)) + cfgerror("Error parsing resume threshold (blocks)"); + if (b_thresh == 0) cfgerror("Invalid resume threshold (blocks)"); for (; isspace(*sp); sp++) {;}@@ -470,7 +472,9 @@ int main(int argc, char *argv[]) f_thresh = ULLONG_MAX; } else { f_thresh = strtoul(sp, &sp, 10); - if (*sp || f_thresh == 0) + if (*sp) + cfgerror("Error parsing resume threshold (files)"); + if (f_thresh == 0) cfgerror("Invalid resume threshold (files)");
Quoted 2 lines
David
I'm fine with either; I gave my R-B regardless. I'm fine with this change, too.