chunkd: add support for multiple key/value tables
PATCH 1/1 chunkd: add support for multiple key/value tables
This is fully fleshed out and working, but I have not committed it yet, in case there is feedback or major objections.
The following patch presents the last major chunkd conceptual change I felt was needed in the chunkd API: multiple key/value tables.
Applications and users will often want to be partitioned away from each other, or simply have a need for something other than simply a flat namespace. With this change, chunkd's API is now much like Amazon S3, which has a shared namespace for buckets, and then each bucket has its own namespace.
This is made possible with a single API addition, TABLE OPEN.
Here is what a chunkd session now looks like, with this patch:
LOGIN(user="jgarzik") TABLE-OPEN(name="tabled") GET... GET... GET... PUT... PUT... PUT...
The corresponding C API call is stc_table_open().
With this change, some of my upper layer projects (NFS, SQL) become more manageable. This change enables me to dedicate several tables to each application and/or user.
configure.ac | 1 doc/chcli.cfg | 3 + include/chunk_msg.h | 4 + include/chunkc.h | 8 ++ lib/chunkdc.c | 42 ++++++++++++++ server/Makefile.am | 6 +- server/be-fs.c | 152 ++++++++++++++++++++++++++++++++++++++++++++------- server/chunkd.h | 15 +++-- server/object.c | 7 +- server/server.c | 64 +++++++++++++++++++-- test/auth.c | 6 ++ test/basic-object.c | 3 + test/it-works.c | 9 +++ test/large-object.c | 3 + test/lotsa-objects.c | 3 + test/nop.c | 3 + test/test.h | 2 tools/chcli.c | 38 +++++++++++- 18 files changed, 332 insertions(+), 37 deletions(-)
configure.ac +1 -0
@@ -79,6 +79,7 @@ AC_CHECK_LIB(event, event_base_new, EVENT_LIBS=-levent, AC_CHECK_LIB(argp, argp_parse, ARGP_LIBS=-largp) AC_CHECK_LIB(socket, bind, SOCKET_LIBS=-lsocket) PKG_CHECK_MODULES(CLDC, libcldc) +PKG_CHECK_MODULES(TOKYOCABINET, tokyocabinet) dnl ----------------------------- dnl Check for cld program, used
doc/chcli.cfg +3 -0
@@ -9,6 +9,9 @@ ## provide the host:port pair of the chunkd service # host=127.0.0.1:9191 +## provide the initial table to open and communicate with +# table=my_table_name + ## provide the username and secret key password for authentication. ## password is ready from CHCLI_PASSWORD env var, if not supplied here. # username=guest
include/chunk_msg.h +4 -0
@@ -21,6 +21,8 @@ enum chunksrv_ops {CHO_DEL =4 ,CHO_LIST =5 ,CHO_LOGIN =6 , +CHO_TABLE_OPEN =7 , + CHO_TABLE_DEL = 8, }; enum chunk_errcode {
@@ -32,10 +34,12 @@ enum chunk_errcode { che_NoSuchKey =5 , che_SignatureDoesNotMatch =6 , che_InvalidKey =7 , + che_InvalidTable = 8, };enum chunk_flags {CHF_SYNC = (1 <<0 ),/* force write to media */ +CHF_TABLE_NEW = (1 <<1 ),/* create table */ }; struct chunksrv_req {
include/chunkc.h +8 -0
@@ -44,6 +44,8 @@ extern void stc_init(void);extern struct st_client *stc_new (const char *service_host,int port,const char *user,const char *secret_key,bool encrypt); +extern bool stc_table_open (struct st_client *stc,const void *key,size_t key_len, +uint32_t flags); extern bool stc_get(struct st_client *stc, const void *key, size_t key_len, size_t (*write_cb)(void *, size_t, size_t, void *),
@@ -103,4 +105,10 @@ static inline bool stc_delz(struct st_client *stc, const char *key) return stc_del(stc, key, strlen(key) + 1); } +static inline bool stc_table_openz (struct st_client *stc,const char *key, +uint32_t flags) +{ +return stc_table_open (stc, key,strlen (key) +1 , flags); +} +#endif /* __STC_H__ */
lib/chunkdc.c +42 -0
48 lines not on this page · raw message
server/Makefile.am +4 -2
@@ -1,5 +1,6 @@ -INCLUDES = -I$(top_srcdir)/include @GLIB_CFLAGS@ @CLDC_CFLAGS@ +INCLUDES = -I$(top_srcdir)/include @GLIB_CFLAGS@ @CLDC_CFLAGS@ \ + @TOKYOCABINET_CFLAGS@ sbin_PROGRAMS = chunkd
@@ -8,4 +9,5 @@ chunkd_SOURCES = chunkd.h \ be-fs.c object.c server.c config.c cldu.c util.c chunkd_LDADD = \ @CLDC_LIBS@ @GLIB_LIBS@ @CRYPTO_LIBS@ \ - @SSL_LIBS@ @EVENT_LIBS@ @ARGP_LIBS@ @SOCKET_LIBS@ + @SSL_LIBS@ @EVENT_LIBS@ @ARGP_LIBS@ @SOCKET_LIBS@ \ + @TOKYOCABINET_LIBS@
server/be-fs.c +133 -19
256 lines not on this page · raw message
server/chunkd.h +11 -4
@@ -78,6 +78,9 @@ struct client {char user[CHD_USER_SZ +1 ]; +size_t table_len; +uint32_t table_id; +SSL *ssl;bool read_want_write;bool write_want_read;
@@ -107,6 +110,7 @@ struct client {char netbuf[CLI_DATA_BUF_SZ ];char netbuf_out[CLI_DATA_BUF_SZ ];char key[CHD_KEY_SZ ]; +char table[CHD_KEY_SZ ]; };struct backend_obj {
@@ -192,9 +196,9 @@ struct server { };/* be-fs.c */ -extern struct backend_obj *fs_obj_new (const void *kbuf,size_t klen, +extern struct backend_obj *fs_obj_new (uint32_t table_id,const void *kbuf,size_t klen,enum chunk_errcode *err_code); -extern struct backend_obj *fs_obj_open (const char *user, +extern struct backend_obj *fs_obj_open (uint32_t table_id,const char *user,const void *kbuf,size_t klen,enum chunk_errcode *err_code);extern ssize_t fs_obj_write (struct backend_obj *bo,const void *ptr,size_t len);
@@ -202,11 +206,14 @@ extern ssize_t fs_obj_read(struct backend_obj *bo, void *ptr, size_t len);extern void fs_obj_free (struct backend_obj *bo);extern bool fs_obj_write_commit (struct backend_obj *bo,const char *user,const char *hashstr,bool sync_data); -extern bool fs_obj_delete (const char *user, +extern bool fs_obj_delete (uint32_t table_id,const char *user,const void *kbuf,size_t klen,enum chunk_errcode *err_code); -extern GList *fs_list_objs (const char *user);extern ssize_t fs_obj_sendfile (struct backend_obj *bo,int out_fd,size_t len); +extern GList *fs_list_objs (uint32_t table_id,const char *user); +extern bool fs_table_open (const char *user,const void *kbuf,size_t klen, +bool create_tbl,uint32_t *table_id, +enum chunk_errcode *err_code);/* object.c */ extern bool object_del (struct client *cli);
server/object.c +4 -3
@@ -30,7 +30,8 @@ bool object_del(struct client *cli)resp_init_req (resp, &cli->creq); - rcb =fs_obj_delete (cli->user, cli->key, cli->key_len, &err); + rcb =fs_obj_delete (cli->table_id, cli->user, + cli->key, cli->key_len, &err);if (!rcb)return cli_err (cli, err, true);
@@ -196,7 +197,7 @@ bool object_put(struct client *cli)if (!user)return cli_err (cli, che_AccessDenied, true); - cli->out_bo =fs_obj_new (cli->key, cli->key_len, &err); + cli->out_bo =fs_obj_new (cli->table_id, cli->key, cli->key_len, &err);if (!cli->out_bo)return cli_err (cli, err, true);
@@ -286,7 +287,7 @@ bool object_get(struct client *cli, bool want_body)resp_init_req (&get_resp->resp, &cli->creq); - cli->in_obj = obj =fs_obj_open (cli->user, cli->key, + cli->in_obj = obj =fs_obj_open (cli->table_id, cli->user, cli->key, cli->key_len, &err);if (!obj) {free (get_resp);
server/server.c +59 -5
114 lines not on this page · raw message
test/auth.c +6 -0
@@ -29,9 +29,15 @@ static void test(bool encrypt) stc1 =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc1); + rcb =stc_table_openz (stc1,TEST_TABLE ,0 ); +OK (rcb); + stc2 =stc_new (TEST_HOST , port,TEST_USER2 ,TEST_USER2_KEY , encrypt);OK (stc2); + rcb =stc_table_openz (stc2,TEST_TABLE ,0 ); +OK (rcb); +/* store object 1 */ rcb =stc_put_inlinez (stc1, key1, val1,strlen (val1),0 );OK (rcb);
test/basic-object.c +3 -0
@@ -27,6 +27,9 @@ static void test(bool encrypt) stc =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc); + rcb =stc_table_openz (stc,TEST_TABLE ,0 ); +OK (rcb); +/* store object */ rcb =stc_put_inlinez (stc, key, val,strlen (val),0 );OK (rcb);
test/it-works.c +9 -0
@@ -21,6 +21,15 @@ static void test(bool ssl) stc =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , ssl);OK (stc); +/* +* we must supply CHF_TABLE_NEW on the first iteration of +* this test, because we are the first test in the testsuite, +* and must create the database to be used by all other tests. +*/ + rcb =stc_table_openz (stc,TEST_TABLE , + ssl ?0 :CHF_TABLE_NEW ); +OK (rcb); + rcb =stc_ping (stc);OK (rcb);
test/large-object.c +3 -0
@@ -93,6 +93,9 @@ static void test(bool encrypt) stc =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc); + rcb =stc_table_openz (stc,TEST_TABLE ,0 ); +OK (rcb); +sync ();gettimeofday (&ta,NULL );
test/lotsa-objects.c +3 -0
@@ -33,6 +33,9 @@ static void test(int n_objects, bool encrypt) stc =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc); + rcb =stc_table_openz (stc,TEST_TABLE ,0 ); +OK (rcb); +fprintf (stderr," lotsa-objects syncing...\n" );sync ();
test/nop.c +3 -0
@@ -28,6 +28,9 @@ static void test(int n_nops, bool encrypt) stc =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc); + rcb =stc_table_openz (stc,TEST_TABLE ,0 ); +OK (rcb); +gettimeofday (&ta,NULL );/* send NOP messages */
test/test.h +2 -0
@@ -8,6 +8,8 @@#define TEST_HOST "localhost" +#define TEST_TABLE "test" +#define TEST_USER "testuser"#define TEST_USER_KEY "testuser"
tools/chcli.c +34 -4
@@ -36,6 +36,8 @@ static struct argp_option options[] = { "Send GET output to FILE, rather than stdout" }, {"ssl" ,'S' ,NULL ,0 , "Enable SSL channel security" }, + { "table", 't', "TABLE", 0, + "Set table for storage and retrieval" }, { "user", 'u', "USER", 0, "Set username to USER" }, { "verbose", 'v', NULL, 0,
@@ -43,6 +45,8 @@ static struct argp_option options[] = { {"list-cmds" ,1001 ,NULL ,0 , "List supported commands" }, + { "create", 1002, NULL, 0, + "Create new table (required, if table does not exist)" }, { } };
@@ -78,6 +82,9 @@ static char *password;static char *output_fn;static char *key_data;static gsize key_data_len; +static char *table_name; +static size_t table_name_len; +static bool table_create;static char *password_env ="CHCLI_PASSWORD" ;static bool chcli_verbose;static bool use_ssl;
@@ -197,6 +204,10 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state)free (s); } + table_name =g_key_file_get_string (config,"global" ,"table" , +NULL ); +if (table_name) + table_name_len =strlen (table_name) +1 ; password =g_key_file_get_string (config,"global" ,"password" ,NULL );
@@ -245,16 +256,23 @@ static error_t parse_opt (int key, char *arg, struct argp_state *state)case 'o' : output_fn = arg;break ; -case 'v' : - chcli_verbose = true; -break ;case 'S' : use_ssl = true;break ; +case 't' : + table_name = arg; + table_name_len =strlen (arg) +1 ; +break ; +case 'v' : + chcli_verbose = true; +break ;case 1001 :/* --list-cmds */ show_cmds ();break ; +case 1002 :/* --create */ + table_create = true; +break ;case ARGP_KEY_ARG :if (cmd_mode !=CHC_NONE )
@@ -298,6 +316,15 @@ static struct st_client *chcli_stc_new(void) stc->verbose = chcli_verbose; +if (!stc_table_open (stc, table_name, table_name_len, + table_create ?CHF_TABLE_NEW :0 )) { +fprintf (stderr,"%s:%u: failed to open table\n" , + host->name, + host->port); +stc_free (stc); +return NULL ; + } +return stc; }
@@ -527,7 +554,10 @@ int main (int argc, char *argv[])fprintf (stderr,"no host specified\n" );return 1 ; } - +if (!table_name || !table_name_len) { +fprintf (stderr,"no table name specified\n" ); +return 1 ; + }if (strlen (username) ==0 ) {fprintf (stderr,"no username specified\n" );return 1 ;
Quoted 3 lines · Jeff Garzik
LOGIN(user="jgarzik") TABLE-OPEN(name="tabled") GET...
2 more turnarounds per session? Brilliant!
The theory behind this is sound: let's not saddle chunkd with caching authentication results, which is ineffective anyway, but provide a way for application to amortize the cost of authentication over a number of requests explicitly. But in practice it means tabled needs to keep inactive sessions open, which is a chunk of code for me to write (and debug!). I guess I'll do it in a few months...
test/auth.c static void test(bool encrypt) · line 36
Quoted 9 lines
test/auth.c +3
-0@@ -29,9 +29,15 @@ static void test(bool encrypt) stc1 =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc1); + rcb =stc_table_openz (stc1,TEST_TABLE ,0 ); +OK (rcb); + stc2 =stc_new (TEST_HOST , port,TEST_USER2 ,TEST_USER2_KEY , encrypt);OK (stc2);
Having a default table? Naah, those lazy application programmers have it too easy already!
Again, from the point of view of chunkd, this makes complete sense:
why carry an extra (default) table in cases when application does in fact set its own tables, right?
test/it-works.c static void test(bool ssl) · line 31
Quoted 8 lines
test/it-works.c +8
-0+/* +* we must supply CHF_TABLE_NEW on the first iteration of +* this test, because we are the first test in the testsuite, +* and must create the database to be used by all other tests. +*/ + rcb =stc_table_openz (stc,TEST_TABLE , + ssl ?0 :CHF_TABLE_NEW ); +OK (rcb);
You've got to be kidding me. How is tabled supposed to know that the request it's making is "first"?! I guess I have to supply CHF_TABLE_NEW to every call now, or else retry if InvalidTable is returned, I haven't decided what workaround to apply yet.
-- Pete
Quoted 6 lines · Pete Zaitcev
Quoted 3 lines · Jeff Garzik
LOGIN(user="jgarzik") TABLE-OPEN(name="tabled") GET...
Quoted 8 lines
2 more turnarounds per session? Brilliant!
The theory behind this is sound: let's not saddle chunkd with caching authentication results, which is ineffective anyway, but provide a way for application to amortize the cost of authentication over a number of requests explicitly. But in practice it means tabled needs to keep inactive sessions open, which is a chunk of code for me to write (and debug!). I guess I'll do it in a few months...
chunkd protocol was never intended to be a connect+request+disconnect model... HTTP 1.0 proved that was a bad model, which is why the world moved on to pipelined, multiple-request protocols. connect+request+disconnect protocols waste kernel, IPVS, firewall and router resources, and are distinctly network unfriendly.
So yeah... tabled does need to keep chunkd sessions open, after a chunkd request completes. That was always true, regardless of the multi-kv API change in $Subject.
As a matter of fact, libchunkdc is actually a limiting factor here: even though the chunkd network protocol is pipeline-able, libchunkdc always waits for a response before returning control back to the application. That is not strictly necessary: an application could choose to submit 10 'DEL' requests in a single write(2), and then wait for 10 responses from the server, if it so wished.
Quoted 13 lines
test/auth.c static void test(bool encrypt) · line 36
Quoted 9 lines
test/auth.c +3
-0@@ -29,9 +29,15 @@ static void test(bool encrypt) stc1 =stc_new (TEST_HOST , port,TEST_USER ,TEST_USER_KEY , encrypt);OK (stc1); + rcb =stc_table_openz (stc1,TEST_TABLE ,0 ); +OK (rcb); + stc2 =stc_new (TEST_HOST , port,TEST_USER2 ,TEST_USER2_KEY , encrypt);OK (stc2);Having a default table? Naah, those lazy application programmers have it too easy already!
Quoted 3 lines
Again, from the point of view of chunkd, this makes complete sense:
why carry an extra (default) table in cases when application does in fact set its own tables, right?
If people want a default table, I can put it in. MySQL tries to connect to a database $Username, if database name is not supplied, for example.
But yes, from point of view of chunkd simplicity, no-default-table is certainly more simple, which makes me reluctant to add it.
If the separate API call is bothersome, we could pass table name to stc_new().
Quoted 5 lines
Quoted 5 lines
test/it-works.c +5
-0+/* +* we must supply CHF_TABLE_NEW on the first iteration of +* this test, because we are the first test in the testsuite, +* and must create the database to be used by all other tests. +*/
Quoted 8 lines
test/it-works.c static void test(bool ssl) · line 31
Quoted 3 lines
test/it-works.c +3
-0+ rcb =stc_table_openz (stc,TEST_TABLE , + ssl ?0 :CHF_TABLE_NEW ); +OK (rcb);You've got to be kidding me. How is tabled supposed to know that the request it's making is "first"?! I guess I have to supply
CHF_TABLE_NEWto every call now, or else retry if InvalidTable is returned, I haven't decided what workaround to apply yet.
A fair question... It seemed logical to create the table at the time a new chunkd node comes online, and that the application would want to always run in normal mode WITHOUT CHF_TABLE_NEW -- thus making a table's unexpected absence a hard error, mirroring real life.
Easy alternatives include (a) create on demand and never worry about this detail, and (b) add an 'exclusive' flag analagous to O_EXCL. This complements CHF_TABLE_NEW, which is analagous to O_CREAT.
Jeff
Quoted 2 lines · Jeff Garzik
But yes, from point of view of chunkd simplicity, no-default-table is certainly more simple, which makes me reluctant to add it.
English parse error :)
no-default-table is more simple, and imposes zero policy on the namespace, which makes me reluctant to add default-table.
Jeff
chunkd design genesis, storage tech, and support for multiple key/value tables
You wrote this insightful and pointed comment on IRC...
Quoted 3 lines
Comparing with "every k/v service out there" assumes that you're growing a generic key/value service out of Chunk. You're essentially admitting it openly.
This is an excellent point to raise. So let the "begin at the beginning", cover the chunkd design thought process, and hope to explain how this matches up.
Let us consider storage technology, at the level I'm used to: ATA, SCSI, and nbd protocols.
For decades, storage has been a run of fixed-length records (sectors and blocks), with the following API:
key = offset + data length
<-- "key" is minimum amount of data required to
uniquely describe a run of data
PUT key, data
data = GET key
Now the world has figured out giving a storage device the flexibility to manage data on a per-object granular basis simplifies applications, and gives underlying storage more ability to optimize. Thus was born the object-based storage device (SCSI OSD), with the API
key = 64-bit object id
PUT key, data, data length
data, data length = GET key
A key design decision of Project Hail was to follow this object-based storage model, when considering the two alternatives:
1) Build cloud apps on top of multple block devices. My conclusion: this is undesirable for the same reason why sector-based storage is undesirable: applications want more granularity, and with sector-based systems, must build their own filesystem-like data structures just to keep their own objects separated from one another.
2) Build cloud apps on top of filesystems. I think(?) GlusterFS is taking this route. This approach is workable, but may create a lot of unnecessary overhead. Filesystem protocols are much more complicated than storage protocols, in particular.
Object-based storage devices sit in the middle: not as complex as filesystems, but more useful than sector-based storage.
chunkd is thus designed to be a simple, straightforward, easy-to-use replacement for SCSI OSD, which has already been proven useful in distributed storage (Lustre, pNFS).
That is why chunkd originally used fixed-length hexidecimal keys: It was modelled on the SCSI OSD object id. However, it quickly became evident in practice that EVERY chunkd application would create its own scheme to map internal_object_id to chunkd_object_id.
Thus, moving to generic key/value storage actually simplified applications, by eliminating that mapping.
However, one glaring difference from SCSI OSD was chunkd's lack of administrative partitions. SCSI OSDs provide "partitions" within each logical unit (LUN), each of contains a set of objects within a single object id namespace. Therefore, if you consider SCSI OSD object id as the key, then SCSI OSD definitely has multiple key/value tables.
As you pointed out on IRC, it is possible to create administrative partitioning by running multiple chunkd instances.
But I think the Real World(tm) has shown that in-protocol partitioning of object namespace is the way to go. Being able to create and destroy partitions within the protocol, on-demand, has a lot of value.
So, just as SCSI OSD has
[ target + logical unit + ] partition + object
With chunkd we can have
[ host + port + ] table + object
Amazon S3 has buckets. Pretty much every protocol in production tends to have some sort of administrative separation ability.
Jeff
Re: chunkd design genesis, storage tech, and support for multiple key/value tables
Quoted 3 lines · Jeff Garzik
Now the world has figured out giving a storage device the flexibility to manage data on a per-object granular basis simplifies applications, and gives underlying storage more ability to optimize.
This is a sleught of hand by OSD vendors, interested in selling for more dollars per gigabyte.
Quoted 2 lines
Thus, moving to generic key/value storage actually simplified applications, by eliminating that mapping.
You're so sure about this, I wonder where it comes from.
The fact in case of tabled is, it must maintain a database of keys of its own, primarily because (a) it cannot afford round-trips into Chunk for every operation, and (b) to locate the chunks. Both of these databases may be in RAM, but it does not make them non-existing.
Quoted 5 lines
However, one glaring difference from SCSI OSD was chunkd's lack of administrative partitions. SCSI OSDs provide "partitions" within each logical unit (LUN), each of contains a set of objects within a single object id namespace. Therefore, if you consider SCSI OSD object id as the key, then SCSI OSD definitely has multiple key/value tables.
This is a completely bogus analogy. OSD vendors want to push their wares into PC space, where one unit is all a computer has. But in the cloud we have thousands of Chunk nodes per each application. That is your partitioning right there: it's called <Cell></Cell>.
Look, I would not mind if all this partition stuff was free, but it's not. You decided to embed a partition into a session, so
- There's a round trip that you excuse by telling applications to keep long-living connections, thanks a lot
- requests to different partitions cannot be pipelined (well, not easily).
-- Pete