chunkd: add self-checking
PATCH 1/1 chunkd: add self-checking
This patch adds the self-check to Chunk. With it, the daemon can rescan all of its keys and drop those that fail to match their own checksums or throw an I/O error. Objects that are found found faulty are made invisible to applications (back-end files are renamed, so that bad blocks are not reused). This is intended to work in concert with applications that store redundant copies of their data objects.
This patch includes the part that tracks the active I/O, since the self-check is the only user of it, so it made little sense to separate the two. We have to track the I/O so that self-check does not mistakenly assume a partially stored object to be faulty and kills it.
Running the self-check can adversely affect performance. As a crude way to limit the problem, we limit the load to one check thread only. Still, as anyone who had mlocate or Beagle running in their desktop knows, the biggest issue is not additional I/O as such, but the blowing away the page cache and dentries in kernel. Also, our scheduling is between rudimentary and non-existing. We only provide a looping check with a randomized delay and an external control to start and verify the running of the checking.
Therefore, to avoid surprises with sudden loss of objects and with performance anomalies, periodic self-check defaults to off.
Still, self-check is an intergral part of the daemon, so we include unit tests for both the I/O tracking and self-check itself.
--- doc/setup.txt | 10 + include/Makefile.am | 2 include/chunk-private.h | 4 include/chunk_msg.h | 21 ++ include/chunkc.h | 4 include/objcache.h | 75 ++++++++ lib/chunkdc.c | 82 +++++++++ server/Makefile.am | 3 server/be-fs.c | 130 ++++++++++++++ server/chunkd.h | 28 +++ server/cldu.c | 1 server/config.c | 18 ++ server/objcache.c | 138 +++++++++++++++ server/object.c | 9 + server/selfcheck.c | 293 +++++++++++++++++++++++++++++++++ server/server.c | 135 +++++++++++++++ test/.gitignore | 2 test/Makefile.am | 7 test/objcache-unit.c | 64 +++++++ test/selfcheck-unit.c | 334 ++++++++++++++++++++++++++++++++++++++ test/test.h | 2 tools/chcli.c | 116 +++++++++++-- 22 files changed, 1454 insertions(+), 24 deletions(-)
commit 6991325fa6c3dd6518f386cd08bbecbdb558295c
Author: Master <[email protected]> Date: Tue Mar 2 11:04:56 2010 -0700
Self-check with a simplified check status.
doc/setup.txt +10 -0
@@ -104,6 +104,16 @@ _cld._udp.phx2.ex.com has SRV record 10 50 8081 maika.phx2.ex.com. <Rack>F3R18</Rack> </Geo> +*) configure the self-check period, if desired. Note that the actual amount + of time that the daemon sleeps between self-check sweeps is randomized + in order to prevent the convoy effect, so the value below is approximate. + + <!-- 30 * 60 * 60 == 108000 seconds --> + <SelfCheckPeriod>108000</SelfCheckPeriod> + + The default is zero, which disables the periodic self-check. It may be + appropriate if the application performs its own end-to-end checking. + *) start daemon (it will put itself into the background) with the configuration file just created:
include/Makefile.am +1 -1
@@ -1,5 +1,5 @@ include_HEADERS = chunkc.h chunk_msg.h -EXTRA_DIST = elist.h chunk_msg.h chunksrv.h chunk-private.h +EXTRA_DIST = elist.h chunk_msg.h chunksrv.h chunk-private.h objcache.h
include/chunk-private.h +4 -0
@@ -52,4 +52,8 @@ static inline uint64_t cpu_to_le64(uint64_t val) return GUINT64_TO_LE(val); } +#define MDB_TPATH_FMT "%s/%X" +#define BAD_TPATH_FMT "%s/bad" +#define PREFIX_LEN 3 +#endif /* __CHUNK_PRIVATE_H__ */
include/chunk_msg.h +21 -0
@@ -40,6 +40,8 @@ enum chunksrv_ops {CHO_LIST =5 ,CHO_LOGIN =6 ,CHO_TABLE_OPEN =7 , +CHO_CHECK_START =8 , + CHO_CHECK_STATUS = 9, }; enum chunk_errcode {
@@ -52,6 +54,7 @@ enum chunk_errcode { che_SignatureDoesNotMatch =6 , che_InvalidKey =7 , che_InvalidTable =8 , + che_Busy = 9, }; enum chunk_flags {
@@ -86,4 +89,22 @@ struct chunksrv_resp_get { uint64_t mtime; }; +enum chunk_check_state { + chk_Off, + chk_Idle, + chk_Active +}; + +struct chunk_check_status { +uint8_t state;/* enum chunk_check_state */ +uint8_t pad[3 ]; +uint32_t count;/* lifetime */ +uint64_t lastdone;/* UTC */ +}; + +struct chunksrv_resp_chkstat { +struct chunksrv_resp resp; +struct chunk_check_status chkstat; +}; +#endif /* __CHUNK_MSG_H__ */
include/chunkc.h +4 -0
@@ -89,6 +89,10 @@ extern bool stc_put_inline(struct st_client *stc, const void *key,extern bool stc_del (struct st_client *stc,const void *key,size_t key_len);extern bool stc_ping (struct st_client *stc); +extern bool stc_check_poke (struct st_client *stc); +extern bool stc_check_status (struct st_client *stc, +struct chunk_check_status *out); +extern struct st_keylist *stc_keys (struct st_client *stc);extern int stc_readport (const char *fname);
include/objcache.h new file +75 -0
75 lines not on this page · raw message
lib/chunkdc.c +82 -0
88 lines not on this page · raw message
server/Makefile.am +2 -1
@@ -6,7 +6,8 @@ sbin_PROGRAMS = chunkd chunkd_SOURCES = chunkd.h \ ../lib/chunksrv.c \ - be-fs.c object.c server.c config.c cldu.c util.c + be-fs.c object.c server.c selfcheck.c config.c cldu.c util.c \ + objcache.c chunkd_LDADD = \ @CLDC_LIBS@ @GLIB_LIBS@ @CRYPTO_LIBS@ \ @SSL_LIBS@ @EVENT_LIBS@ @TOKYOCABINET_LIBS@
server/be-fs.c +125 -5
159 lines not on this page · raw message
server/chunkd.h +28 -0
@@ -28,6 +28,7 @@#include <chunk_msg.h> #include <hail_log.h> #include <tchdb.h> +#include <objcache.h> #ifndef ARRAY_SIZE #define ARRAY_SIZE (arr) (sizeof(arr) / sizeof((arr)[0]))
@@ -118,6 +119,7 @@ struct client {uint64_t out_len;struct backend_obj *out_bo; +struct objcache_entry *out_ce;uint64_t in_len;struct backend_obj *in_obj;
@@ -188,8 +190,21 @@ struct server_socket {const struct listen_cfg *cfg; }; +enum chk_cmd { +CHK_CMD_EXIT , +CHK_CMD_RESCAN +}; + +enum chk_state { +CHK_ST_OFF , +CHK_ST_INIT , +CHK_ST_IDLE , +CHK_ST_RUNNING , +}; +struct server {unsigned long flags;/* SFL_xxx above */ +GMutex *bigmutex;char *config;/* master config file */ char *pid_file;/* PID file */
@@ -208,12 +223,19 @@ struct server {char *group;uint32_t nid;struct geo loc; +time_t chk_period; +int chk_pipe[2 ];TCHDB *tbl_master; +struct objcache actives;struct server_stats stats;/* global statistics */ +enum chk_state chk_state; +time_t chk_done; }; +#define MDB_TABLE_ID "__chunkd_table_id" +extern struct hail_log cldu_hail_log;/* be-fs.c */
@@ -242,6 +264,7 @@ extern bool fs_obj_write_commit(struct backend_obj *bo, const char *user,extern bool fs_obj_delete (uint32_t table_id,const char *user,const void *kbuf,size_t klen,enum chunk_errcode *err_code); +extern int fs_obj_disable (const char *fn);extern ssize_t fs_obj_sendfile (struct backend_obj *bo,int out_fd,size_t len);extern int fs_list_objs_open (struct fs_obj_lister *t,const char *root_path,uint32_t table_id);
@@ -254,6 +277,7 @@ extern GList *fs_list_objs(uint32_t table_id, const char *user);extern bool fs_table_open (const char *user,const void *kbuf,size_t klen,bool tbl_creat,bool excl_creat,uint32_t *table_id,enum chunk_errcode *err_code); +extern int fs_obj_do_sum (const char *fn,unsigned int klen,char **csump);/* object.c */ extern bool object_del (struct client *cli);
@@ -317,6 +341,10 @@ extern void resp_init_req(struct chunksrv_resp *resp,/* config.c */ extern void read_config (void ); +/* selfcheck.c */ +extern void chk_init (void ); +extern int chk_spawn (time_t period,TCHDB *hdb); +static inline bool use_sendfile (struct client *cli) {#if defined(HAVE_SENDFILE ) && defined(HAVE_SYS_SENDFILE_H )
server/cldu.c +0 -1
@@ -248,7 +248,6 @@ static struct cldc_ops cld_ops = { .timer_ctl = cldu_p_timer_ctl, .pkt_send = cldu_p_pkt_send, .event = cldu_p_event, - .errlog = applog, };/*
server/config.c +18 -0
@@ -442,6 +442,24 @@ static void cfg_elm_end (GMarkupParseContext *context, cc->text =NULL ; } +else if (!strcmp (element_name,"SelfCheckPeriod" )) { +if (!cc->text) { +applog (LOG_WARNING ,"SelfCheckPeriod element empty" ); +return ; + } + n =strtol (cc->text,NULL ,10 ); +if (n <0 || n >=LONG_MAX ) { +applog (LOG_ERR ,"SelfCheckPeriod '%s' is invalid" , + cc->text); +free (cc->text); + cc->text =NULL ; +return ; + } + chunkd_srv.chk_period = n; +free (cc->text); + cc->text =NULL ; + } +else {applog (LOG_WARNING ,"Unknown element \"%s\"" , element_name); }
server/objcache.c new file +138 -0
138 lines not on this page · raw message
server/object.c +9 -0
@@ -73,6 +73,10 @@ void cli_out_end(struct client *cli)fs_obj_free (cli->out_bo); cli->out_bo =NULL ; } +if (cli->out_ce) { +objcache_put (&chunkd_srv.actives, cli->out_ce); + cli->out_ce =NULL ; + }free (cli->out_user); cli->out_user =NULL ;
@@ -217,6 +221,11 @@ bool object_put(struct client *cli)if (!user)return cli_err (cli, che_AccessDenied, true); + cli->out_ce =objcache_get_dirty (&chunkd_srv.actives, + cli->key, cli->key_len); +if (!cli->out_ce) +return cli_err (cli, che_InternalError, true); + cli->out_bo =fs_obj_new (cli->table_id, cli->key, cli->key_len, &err);if (!cli->out_bo)return cli_err (cli, err, true);
server/selfcheck.c new file +293 -0
293 lines not on this page · raw message
server/server.c +134 -1
204 lines not on this page · raw message
test/.gitignore +2 -0
@@ -4,6 +4,8 @@ it-works large-object lotsa-objects nop +objcache-unit +selfcheck-unit .libs libtest.a
test/Makefile.am +6 -1
@@ -15,6 +15,7 @@ EXTRA_DIST = \ ssl-key.pem ssl-cert.pem TESTS = \ + objcache-unit \ prep-db \ start-daemon \ pid-exists \
@@ -25,11 +26,12 @@ TESTS = \ auth \ large-object \ lotsa-objects \ + selfcheck-unit \ stop-daemon \ clean-db check_PROGRAMS = auth basic-object it-works large-object \ - lotsa-objects nop + lotsa-objects nop objcache-unit selfcheck-unit TESTLDADD = ../lib/libchunkdc.la \ libtest.a \
@@ -41,6 +43,9 @@ it_works_LDADD = $(TESTLDADD) large_object_LDADD = $(TESTLDADD) lotsa_objects_LDADD = $(TESTLDADD) nop_LDADD = $(TESTLDADD) +selfcheck_unit_LDADD = $(TESTLDADD) + +objcache_unit_LDADD = @GLIB_LIBS@ noinst_LIBRARIES = libtest.a
test/objcache-unit.c new file +64 -0
64 lines not on this page · raw message
test/selfcheck-unit.c new file +334 -0
334 lines not on this page · raw message
test/test.h +2 -0
@@ -38,6 +38,8 @@#define TEST_PORTFILE "chunkd.port"#define TEST_PORTFILE_SSL "chunkd-ssl.port" +#define TEST_CHUNKD_CFG "server-test.cfg" +#define OK (expr) \ do { \ if (!(expr)) { \
tools/chcli.c +102 -14
200 lines not on this page · raw message
Quoted 45 lines · Pete Zaitcev
This patch adds the self-check to Chunk. With it, the daemon can rescan all of its keys and drop those that fail to match their own checksums or throw an I/O error. Objects that are found found faulty are made invisible to applications (back-end files are renamed, so that bad blocks are not reused). This is intended to work in concert with applications that store redundant copies of their data objects.
This patch includes the part that tracks the active I/O, since the self-check is the only user of it, so it made little sense to separate the two. We have to track the I/O so that self-check does not mistakenly assume a partially stored object to be faulty and kills it.
Running the self-check can adversely affect performance. As a crude way to limit the problem, we limit the load to one check thread only. Still, as anyone who had mlocate or Beagle running in their desktop knows, the biggest issue is not additional I/O as such, but the blowing away the page cache and dentries in kernel. Also, our scheduling is between rudimentary and non-existing. We only provide a looping check with a randomized delay and an external control to start and verify the running of the checking.
Therefore, to avoid surprises with sudden loss of objects and with performance anomalies, periodic self-check defaults to off.
Still, self-check is an intergral part of the daemon, so we include unit tests for both the I/O tracking and self-check itself.
Signed-off-by Pete Zaitcev<[email protected]>--- doc/setup.txt | 10 + include/Makefile.am | 2 include/chunk-private.h | 4 include/chunk_msg.h | 21 ++ include/chunkc.h | 4 include/objcache.h | 75 ++++++++ lib/chunkdc.c | 82 +++++++++ server/Makefile.am | 3 server/be-fs.c | 130 ++++++++++++++ server/chunkd.h | 28 +++ server/cldu.c | 1 server/config.c | 18 ++ server/objcache.c | 138 +++++++++++++++ server/object.c | 9 + server/selfcheck.c | 293 +++++++++++++++++++++++++++++++++
Quoted 8 lines
server/server.c | 135 +++++++++++++++ test/.gitignore | 2 test/Makefile.am | 7 test/objcache-unit.c | 64 +++++++ test/selfcheck-unit.c | 334 ++++++++++++++++++++++++++++++++++++++ test/test.h | 2 tools/chcli.c | 116 +++++++++++-- 22 files changed, 1454 insertions(+), 24 deletions(-)
applied... looks mostly OK. Things like self-check period still need to be removed, and a few other minor quibbles. But let's go ahead and get this in, then tackle those things.