tabled: Add replication daemon
PATCH 1/1 tabled: Add replication daemon
This patch adds what amounts to a background process that maintains redundancy for object data. It is far from the complete solution. For one thing, it does not verify checksums. But it's a start.
There's no way to turn this off, by intention. The whole thing must work very reliably, not steal too much from benchmarks (but if it does, it's only honest to take the hit). It is indispensible. However, there's a plan to add useful monitoring of jobs and other state, such as available nodes.
This implementation uses a separate thread.
--- configure.ac | 2 server/Makefile.am | 4 server/object.c | 6 server/replica.c | 855 +++++++++++++++++++++++++++++++++++++++++++ server/server.c | 21 - server/storage.c | 72 +++ server/tabled.h | 24 + 7 files changed, 966 insertions(+), 18 deletions(-)
Can go on top of tabled with CLD de-timerization or without.
commit 9a543b33b53f4ce67fc104270eedbd064d44aa1a
Author: Master <[email protected]> Date: Thu Nov 26 19:28:57 2009 -0700
First stab at replication (threaded).
configure.ac +1 -1
@@ -100,7 +100,7 @@ dnl -------------------------- dnl autoconf output generation dnl -------------------------- -AM_PATH_GLIB_2_0(2.0.0) +AM_PATH_GLIB_2_0(2.0.0, , exit 1, gthread) AM_PATH_XML2(2.6.0, , [AC_MSG_ERROR([Missing required XML2 >= 2.6.0])]) LIBCURL_CHECK_CONFIG(, 7.10.1, ,
server/Makefile.am +2 -2
@@ -4,8 +4,8 @@ INCLUDES = -I$(top_srcdir)/include @GLIB_CFLAGS@ @CHUNKDC_CFLAGS@ @CLDC_CFLAGS@ sbin_PROGRAMS = tabled tdbadm tabled_SOURCES = tabled.h \ - bucket.c object.c server.c storage.c storparse.c cldu.c \ - config.c util.c + bucket.c object.c server.c storage.c storparse.c replica.c \ + cldu.c config.c util.c tabled_LDADD = ../lib/libhttputil.a ../lib/libtdb.a \ @CHUNKDC_LIBS@ @CLDC_LIBS@ @PCRE_LIBS@ @GLIB_LIBS@ \ @CRYPTO_LIBS@ @DB4_LIBS@ @EVENT_LIBS@ @ARGP_LIBS@ @SSL_LIBS@
server/object.c +4 -2
@@ -151,6 +151,7 @@ static void object_unlink(struct db_obj_ent *obj)"object data(%llX) unlink failed on nid %u" , (unsigned long long )GUINT64_FROM_LE (addr->oid), nid); +stor_node_put (stnode); } }
@@ -694,7 +695,7 @@ static struct open_chunk *open_chunk1(struct storage_node *stnode, goto err_alloc; } - rc =stor_open (ochunk, stnode); + rc =stor_open (ochunk, stnode, tabled_srv.evbase_main);if (rc !=0 ) {applog (LOG_WARNING ,"Cannot open output chunk, nid %u (%d)" , stnode->id, rc);
@@ -1183,6 +1184,7 @@ bool object_get_body(struct client *cli, const char *user, const char *bucket,applog (LOG_DEBUG ,"Selected nid %u for oid %llX" , nid, cli->in_objid); +stor_node_put (stnode);break ; } }
@@ -1191,7 +1193,7 @@ bool object_get_body(struct client *cli, const char *user, const char *bucket,if (!stnode) goto stnode_open_retry; - rc =stor_open (&cli->in_ce, stnode); + rc =stor_open (&cli->in_ce, stnode, tabled_srv.evbase_main);if (rc <0 ) {applog (LOG_WARNING ,"Cannot open input chunk, nid %u (%d)" , stnode->id, rc);
server/replica.c new file +855 -0
855 lines not on this page · raw message
server/server.c +19 -2
@@ -374,6 +374,8 @@ static void stats_dump(void)X (opt_write);applog (LOG_INFO ,"State: TDB %s" , state_name_tdb[tabled_srv.state_tdb]); +stor_stats (); +rep_stats (); }#undef X
@@ -1362,7 +1364,7 @@ static void tdb_state_cb(enum db_event event)* * We don't even bother with registering this callback, just call it by name. * -* The return value is used to re-arm rescan mechanism. +* The return value is used to re-arm storage rescan mechanism. */ int stor_update_cb (void ) {
@@ -1380,8 +1382,15 @@ int stor_update_cb(void)applog (LOG_DEBUG ," NID %u is up" , stn->id); num_up++; stn->up = true; + stn->last_up =time (NULL ); + }else { +if (stn->last_up !=0 && +time (NULL ) >= stn->last_up +CHUNK_REBOOT_TIME ) { +applog (LOG_INFO ," NID %u went down" , stn->id); + } } } +if (num_up <1 ) {applog (LOG_INFO ,"No active storage node(s), waiting" );return num_up;
@@ -1692,6 +1701,7 @@ static void tdb_state_process(enum st_tdb new_state)return ; }add_chkpt_timer (); +rep_start ();net_listen (); } }
@@ -1700,6 +1710,7 @@ int main (int argc, char *argv[]) {error_t aprc;int rc =1 ; +struct event_base *event_base_rep;INIT_LIST_HEAD (&tabled_srv.all_stor); tabled_srv.state_tdb =ST_TDB_INIT ;
@@ -1709,6 +1720,9 @@ int main (int argc, char *argv[])compile_patterns (); +g_thread_init (NULL ); + tabled_srv.bigmutex =g_mutex_new (); +SSL_library_init ();SSL_load_error_strings ();
@@ -1767,7 +1781,8 @@ int main (int argc, char *argv[])signal (SIGTERM , term_signal);signal (SIGUSR1 , stats_signal); -event_init (); + event_base_rep =event_base_new (); + tabled_srv.evbase_main =event_init ();evtimer_set (&tabled_srv.chkpt_timer, tdb_checkpoint,NULL );/* set up server networking */
@@ -1780,6 +1795,8 @@ int main (int argc, char *argv[]) goto err_cld_session; } +rep_init (event_base_rep); +applog (LOG_INFO ,"initialized (%s)" , (tabled_srv.flags &SFL_FOREGROUND )?"fg" :"bg" );
server/storage.c +65 -7
158 lines not on this page · raw message
server/tabled.h +20 -4
@@ -19,9 +19,9 @@* */ -#include <stdbool.h> #include <stdlib.h> +#include <time.h> #include <netinet/in.h> #include <openssl/md5.h> #include <glib.h>
@@ -43,6 +43,9 @@ enum {TABLED_PGSZ_LOCK =4096 ,TABLED_CHKPT_SEC =60 *5 ,/* secs between db4 chkpt */ +TABLED_RESCAN_SEC =60 *3 +7 ,/* secs btw key rescans */ + +CHUNK_REBOOT_TIME =3 *60 ,/* secs to declare chunk dead */ CLI_REQ_BUF_SZ =8192 ,/* buffer for req + hdrs */ CLI_DATA_BUF_SZ = 8192,
@@ -89,13 +92,14 @@ struct storage_node {struct list_head all_link;uint32_t id;bool up; +time_t last_up;unsigned alen;int addr_af;struct sockaddr_in6 addr;char *hostname;/* Only used because stc_new is overly smart. */ -int nchu;/* number of open_chunk */ +int ref;/* number of open_chunk or other */ };typedef bool (*cli_evt_func )(struct client *,unsigned int );
@@ -115,7 +119,8 @@ struct open_chunk {struct st_client *stc;struct storage_node *node;struct list_head link; -struct client *cli; +void *cli;/* usually struct client * */ +struct event_base *evbase;uint64_t wtogo;uint64_t wkey;
@@ -217,6 +222,8 @@ struct listen_cfg {struct server {unsigned long flags;/* SFL_xxx above */ int pid_fd;/* fd of pid_file */ +GMutex *bigmutex; +struct event_base *evbase_main;char *config;/* config file (static) */
@@ -318,7 +325,10 @@ extern int stor_update_cb(void);extern void read_config (void );/* storage.c */ -extern int stor_open (struct open_chunk *cep,struct storage_node *stn); +extern struct storage_node *stor_node_get (struct storage_node *stn); +extern void stor_node_put (struct storage_node *stn); +extern int stor_open (struct open_chunk *cep,struct storage_node *stn, +struct event_base *ev_base);extern int stor_open_read (struct open_chunk *cep,void (*cb)(struct open_chunk *),uint64_t key,uint64_t *psz);
@@ -336,8 +346,14 @@ extern struct storage_node *stor_node_by_nid(uint32_t nid);extern void stor_add_node (uint32_t nid,const char *hostname,const char *portstr,struct geo *locp);extern int stor_node_check (struct storage_node *stn); +extern void stor_stats (void );/* storparse.c */ extern void stor_parse (char *fname,const char *text,size_t len); +/* replica.c */ +extern void rep_init (struct event_base *ev_base); +extern void rep_start (void ); +extern void rep_stats (void ); +#endif /* __TABLED_H__ */
Quoted 10 lines · Pete Zaitcev
This patch adds what amounts to a background process that maintains redundancy for object data. It is far from the complete solution. For one thing, it does not verify checksums. But it's a start.
There's no way to turn this off, by intention. The whole thing must work very reliably, not steal too much from benchmarks (but if it does, it's only honest to take the hit). It is indispensible. However, there's a plan to add useful monitoring of jobs and other state, such as available nodes.
Quoted 3 lines
This implementation uses a separate thread.
Signed-off-by Pete Zaitcev<[email protected]>
Fantastic, thanks for updating this. Will review in-depth during next work session (it's the holidays here, and I've been under the weather to boot).
Jeff
Quoted 10 lines · Pete Zaitcev
This patch adds what amounts to a background process that maintains redundancy for object data. It is far from the complete solution. For one thing, it does not verify checksums. But it's a start.
There's no way to turn this off, by intention. The whole thing must work very reliably, not steal too much from benchmarks (but if it does, it's only honest to take the hit). It is indispensible. However, there's a plan to add useful monitoring of jobs and other state, such as available nodes.
Quoted 3 lines
This implementation uses a separate thread.
Signed-off-by Pete Zaitcev<[email protected]>
The only thing I am unclear on / worried about is the event_base stuff for libevent.
It seems to me you should
a) leave current main-thread libevent code unchanged
b) call event_base_new() for the new thread, passing that newly-created event_base to replica thread events using event_base_set().
event_reinit() seems more appropriate for forks than threads. tabled's needs are simple -- you simply need to create an additional event_base for the new thread.
Other comments (which may be irrelevant given the above):
server/replica.c line 834
Quoted 4 lines
server/replica.c new file +4
-0+/* Statics are a silly way to pass arguments to threads, but oh well. */ + evbase = ev_base; + + scan_thread =g_thread_create (rep_thread_func,NULL , FALSE, &error);
The second argument to g_thread_create() is a user-provided data pointer, passed directly to the newly created thread. No need for a static, AFAICS.
Jeff
Quoted 8 lines · On Thu, 03 Dec 2009 00:42:47 -0500 Jeff Garzik <[email protected]> wrote:
On 11/26/2009 09:39 PM, Pete Zaitcev wrote:
It seems to me you should
a) leave current main-thread libevent code unchanged
b) call
event_base_new()for the new thread, passing that newly-createdevent_baseto replica thread events usingevent_base_set().
Isn't it exactly what I do right now?
Quoted 3 lines
event_reinit()seems more appropriate for forks than threads. tabled's needs are simple -- you simply need to create an additionalevent_basefor the new thread.
It wasn't obvious from the description and anyway, what's the harm? I thought it reopens some /dev/epoll or whatever may be necessary in the OS.
Quoted 8 lines
server/replica.c line 834
Quoted 4 lines
server/replica.c new file +4
-0+/* Statics are a silly way to pass arguments to threads, but oh well. */ + evbase = ev_base; + + scan_thread =g_thread_create (rep_thread_func,NULL , FALSE, &error);The second argument to
g_thread_create()is a user-provided data pointer, passed directly to the newly created thread. No need for a static, AFAICS.
Ignoring the comment for now, the problem is that evbase is needed in the "system context" of a thread, for things like passing it down to stor_open(). That's the main reason why it's static. Threads in Glib have something called "static private", which kinda-sorta works like what we need, but it's a pain.
I guess I can try to use it...
-- Pete
Jeff Garzik
AppliedQuoted 10 lines · Pete Zaitcev
This patch adds what amounts to a background process that maintains redundancy for object data. It is far from the complete solution. For one thing, it does not verify checksums. But it's a start.
There's no way to turn this off, by intention. The whole thing must work very reliably, not steal too much from benchmarks (but if it does, it's only honest to take the hit). It is indispensible. However, there's a plan to add useful monitoring of jobs and other state, such as available nodes.
Quoted 3 lines
This implementation uses a separate thread.
Signed-off-by Pete Zaitcev<[email protected]>
applied, after fixing event_reinit() and comment bugs