chunkd: update backend to support binary keys
PATCH 1/1 chunkd: update backend to support binary keys
(yeah, the summary line needs s/cld:/chunkd:/)
Just committed the following change to chunkd's storage backend API, preparing chunkd for variable-length opaque binary keys.
commit 0a8c3225355de86484eba71279c646f7790d5bb1
Author: Jeff Garzik <[email protected]> Date: Wed Nov 4 00:44:55 2009 -0500
cld: update storage backend to store based on opaque binary key + key length
Change internal structure to utilize an opaque binary key, rather than the existing probably-nul-terminated, fixed length C hexidecimal string key.
Storage backend now names files based on the SHA256 of the key.
The key, in its entirety, is written to each object's header in the filesystem, to assist in recovery.
This work prepares the server for an API and protocol change permitting arbitrary length binary keys.
include/chunk_msg.h +1 -1
@@ -31,7 +31,7 @@ enum errcode { NoSuchKey, PreconditionFailed, SignatureDoesNotMatch, - InvalidCookie, + InvalidKey, }; struct chunksrv_req {
server/be-fs.c +122 -58
359 lines not on this page · raw message
server/chunkd.h +10 -6
@@ -107,7 +107,8 @@ struct client {struct backend_obj {void *private; -char cookie[MAX_COOKIE_LEN +1 ]; +void *key; +size_t key_len;uint64_t size;time_t mtime;
@@ -134,7 +135,8 @@ struct geo {struct volume_entry {unsigned long long size;/* obj size */ time_t mtime;/* obj last-mod time */ -char *name;/* obj id */ +void *key;/* obj id */ +int key_len;char *hash;/* obj SHA1 checksum */ char *owner;/* obj owner username */ };
@@ -183,16 +185,18 @@ struct server { };/* be-fs.c */ -extern struct backend_obj *fs_obj_new (const char *cookie, +extern struct backend_obj *fs_obj_new (const void *kbuf,size_t klen,enum errcode *err_code); -extern struct backend_obj *fs_obj_open (const char *user,const char *cookie, +extern struct backend_obj *fs_obj_open (const char *user, +const void *kbuf,size_t klen,enum errcode *err_code);extern ssize_t fs_obj_write (struct backend_obj *bo,const void *ptr,size_t len);extern ssize_t fs_obj_read (struct backend_obj *bo,void *ptr,size_t len);extern void fs_obj_free (struct backend_obj *bo);extern bool fs_obj_write_commit (struct backend_obj *bo,const char *user,const char *hashstr,bool sync_data); -extern bool fs_obj_delete (const char *user,const char *cookie, +extern bool fs_obj_delete (const char *user, +const void *kbuf,size_t klen,enum errcode *err_code);extern GList *fs_list_objs (const char *user);extern ssize_t fs_obj_sendfile (struct backend_obj *bo,int out_fd,size_t len);
@@ -223,7 +227,7 @@ extern void timer_add(struct timer *timer, time_t expires);extern void timer_del (struct timer *timer);extern time_t timers_run (void );extern char *time2str (char *strbuf,time_t time); -extern void shastr (const unsigned char *digest,char *outstr); +extern void hexstr (const unsigned char *buf,size_t buf_len,char *outstr); static inline void timer_init(struct timer *timer, const char *name, void (*cb)(struct timer *),
server/object.c +8 -6
@@ -17,7 +17,7 @@ static bool object_get_more(struct client *cli, struct client_write *wr,bool object_del (struct client *cli) { -const char *obj_name = cli->creq.key; +const char *obj_key = cli->creq.key;int rc;enum errcode err = InternalError;bool rcb;
@@ -31,7 +31,8 @@ bool object_del(struct client *cli)memcpy (resp, &cli->creq,sizeof (cli->creq)); - rcb =fs_obj_delete (cli->creq.user, obj_name, &err); + rcb =fs_obj_delete (cli->creq.user, obj_key, +strnlen (obj_key,CHD_KEY_SZ ), &err);if (!rcb)return cli_err (cli, err, true);
@@ -78,7 +79,7 @@ static bool object_put_end(struct client *cli) cli->state = evt_recycle;SHA1_Final (md, &cli->out_hash); -shastr (md, hashstr); +hexstr (md,SHA_DIGEST_LENGTH , hashstr); rcb =fs_obj_write_commit (cli->out_bo, cli->out_user, hashstr, cli->out_sync);
@@ -191,7 +192,7 @@ bool object_put(struct client *cli)if (!user)return cli_err (cli, AccessDenied, true); - cli->out_bo =fs_obj_new (key, &err); + cli->out_bo =fs_obj_new (key,strnlen (key,CHD_KEY_SZ ), &err);if (!cli->out_bo)return cli_err (cli, err, true);
@@ -269,7 +270,7 @@ err_out_buf:bool object_get (struct client *cli,bool want_body) { -const char *obj_name = cli->creq.key; +const char *obj_key = cli->creq.key;int rc;enum errcode err = InternalError;struct backend_obj *obj;
@@ -283,7 +284,8 @@ bool object_get(struct client *cli, bool want_body)memcpy (resp, &cli->creq,sizeof (cli->creq)); - cli->in_obj = obj =fs_obj_open (cli->creq.user, obj_name, &err); + cli->in_obj = obj =fs_obj_open (cli->creq.user, obj_key, +strnlen (obj_key,CHD_KEY_SZ ), &err);if (!obj) {free (resp);return cli_err (cli, err, true);
server/server.c +11 -6
@@ -118,9 +118,9 @@ static struct { {"SignatureDoesNotMatch" ,403 , "The calculated request signature does not match your provided one" }, - [InvalidCookie] = - { "InvalidCookie", 400, - "Cookie check failed" }, + [InvalidKey] = + { "InvalidKey", 400, + "Invalid key presented" }, };void applog (int prio,const char *fmt, ...)
@@ -738,13 +738,16 @@ static bool volume_list(struct client *cli) tmpl = res;while (tmpl) { -char timestr[50 ]; +char timestr[50 ], *esc_key;struct volume_entry *ve; ve = tmpl->data; tmpl = tmpl->next; - s =g_markup_printf_escaped ( +/* copy-and-escape key into nul-terminated buffer */ + esc_key =g_markup_escape_text (ve->key, ve->key_len); + + s =g_strdup_printf (" <Contents>\r\n" " <Name>%s</Name>\r\n" " <LastModified>%s</LastModified>\r\n"
@@ -753,7 +756,7 @@ static bool volume_list(struct client *cli)" <Owner>%s</Owner>\r\n" " </Contents>\r\n" , - ve->name, + esc_key,time2str (timestr, ve->mtime), ve->hash, ve->size,
@@ -761,6 +764,8 @@ static bool volume_list(struct client *cli) content =g_list_append (content, s); +free (esc_key); +free (ve->key);free (ve); }
server/util.c +5 -5
@@ -155,17 +155,17 @@ int fsetflags(const char *prefix, int fd, int or_flags)return rc; } -void shastr (const unsigned char *digest,char *outstr) +void hexstr (const unsigned char *buf,size_t buf_len,char *outstr) {static const char hex[] ="0123456789abcdef" ;int i; -for (i =0 ; i <SHA_DIGEST_LENGTH ; i++) { - outstr[i *2 ] = hex[(digest[i] &0xF0 ) >>4 ]; - outstr[(i *2 ) +1 ] = hex[(digest[i] &0x0F ) ]; +for (i =0 ; i < buf_len; i++) { + outstr[i *2 ] = hex[(buf[i] &0xF0 ) >>4 ]; + outstr[(i *2 ) +1 ] = hex[(buf[i] &0x0F ) ]; } - outstr[SHA_DIGEST_LENGTH *2 ] =0 ; + outstr[buf_len *2 ] =0 ; }char *time2str (char *strbuf,time_t src_time)
Quoted 15 lines · Jeff Garzik
(yeah, the summary line needs s/cld:/chunkd:/)
Just committed the following change to chunkd's storage backend API, preparing chunkd for variable-length opaque binary keys.
commit
0a8c3225355de86484eba71279c646f7790d5bb1Author: Jeff Garzik<[email protected]> Date: Wed Nov 4 00:44:55 2009 -0500cld: update storage backend to store based on opaque binary key + key length
Quoted 7 lines
Change internal structure to utilize an opaque binary key, rather than the existing probably-nul-terminated, fixed length C hexidecimal string key.
Storage backend now names files based on the SHA256 of the key.
The key, in its entirety, is written to each object's header in the filesystem, to assist in recovery.
To emphasize, this is a chunkd on-disk database format change.
Jeff