Skip to contentSkip to conversation

tabled: write objects to multiply chunks

PATCH Applied Pete Zaitcev 2 participants 2 messages Sep 25, 2009 – Sep 29, 2009

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 tabled: write objects to multiply chunks

This is a small, but self-contained step toward the data redundancy. The patch adds ability to write data to several Chunk servers. Along the way, it fixes things up so we use asynchronous writes.

N.B. This patch changes the format of tabled's database in order to reflect that object IDs are global in the cell. This is done so the consistency checking and redundancy monitoring and recovery could be run without accessing the tabled's database. In orther words, it's a flag day, the database must be re-created.

include/tdb.h +4 -4
index 51843ba..e12d00e 100644
@@ -30,8 +30,8 @@
 #define DB_OBJ_INLINE        0x1
 
 struct db_obj_addr {
-	uint32_t	nid;			/* 0 == absent */
-	uint64_t	oid;
+	uint64_t oid;
+	uint32_t nidv[MAXWAY];			/* 0 == absent */
 };
 
 struct db_obj_key {
@@ -41,16 +41,16 @@ struct db_obj_key {
 
 struct db_obj_ent {
 	uint32_t	flags;
+	uint32_t	n_str;			/* # attached string pairs */
 	uint64_t	size;
 	uint64_t	mtime;		/* UNIX time, but in microseconds */
 	union {
-		struct db_obj_addr avec[MAXWAY];
+		struct db_obj_addr a;
 		unsigned char indata[INSIZE];
 	} d;
 	char		bucket[64];		/* bucket */
 	char		owner[64];		/* object owner */
 	char		md5[40];		/* data checksum */
-	uint32_t	n_str;			/* # attached string pairs */
 
 	/* array of uint16_t
 	   representing string lengths of HTTP headers.
server/bucket.c +1 -1
index 60e46bc..976d716 100644
@@ -997,7 +997,7 @@ static bool bucket_list_keys(struct client *cli, const char *user,
 		memset(&v, 0, sizeof(v));
 		strcpy(v.md5, obj->md5);
 		if (!(GUINT32_FROM_LE(obj->flags) & DB_OBJ_INLINE))
-			memcpy(&v.addr, &obj->d.avec[0], sizeof(v.addr));
+			memcpy(&v.addr, &obj->d.a, sizeof(v.addr));
 		v.mtime = obj->mtime;
 		v.size = obj->size;
 		free(obj);
server/cldu.c +0 -1
index 706faeb..ea2266f 100644
@@ -198,7 +198,6 @@ static void cldu_event(int fd, short events, void *userdata)
 #if 0
 		if (rc == -ECONNREFUSED) {	/* ICMP tells us */
 			int newactive;
-			/* P3 */ applog(LOG_INFO, "Restarting session");
 			// evtimer_del(&sp->tm);
 			cldc_kill_sess(sp->lib->sess);
 			sp->lib->sess = NULL;
server/object.c +295 -84

510 lines not on this page · raw message

server/server.c +6 -2
index 60d28df..f1c49be 100644
@@ -418,8 +418,9 @@ static void cli_free(struct client *cli)
 
 	/* clean up network socket */
 	if (cli->fd >= 0) {
-		if (event_del(&cli->ev) < 0)
+		if (cli->ev_active && event_del(&cli->ev) < 0)
 			applog(LOG_WARNING, "TCP client event_del");
+		cli->ev_active = false;
 		close(cli->fd);
 	}
 
@@ -444,6 +445,7 @@ static struct client *cli_alloc(void)
 
 	cli->state = evt_read_req;
 	INIT_LIST_HEAD(&cli->write_q);
+	INIT_LIST_HEAD(&cli->out_ch);
 	cli->req_ptr = cli->req_buf;
 	memset(&cli->req, 0, sizeof(cli->req) - sizeof(cli->req.hdr));
 
@@ -1286,6 +1288,7 @@ static void tcp_srv_event(int fd, short events, void *userdata)
 		applog(LOG_WARNING, "tcp client event_add");
 		goto err_out_fd;
 	}
+	cli->ev_active = true;
 
 	/* pretty-print incoming cxn info */
 	memset(host, 0, sizeof(host));
@@ -1393,6 +1396,7 @@ int stor_update_cb(void)
 			if (debugging)
 				applog(LOG_DEBUG, " NID %u is up", stn->id);
 			num_up++;
+			stn->up = true;
 		}
 	}
 	if (num_up < 1) {
@@ -1400,7 +1404,7 @@ int stor_update_cb(void)
 		return num_up;
 	}
 	if (debugging)
-		applog(LOG_DEBUG, "Active storage node(s): %d", stn->id);
+		applog(LOG_DEBUG, "Active storage node(s): %d", num_up);
 
 	/*
 	 * We initiate operations even if there's no redundancy in order
server/storage.c +43 -11

123 lines not on this page · raw message

server/tabled.h +16 -3
index 5f74748..ff0e699 100644
@@ -89,6 +89,7 @@ struct geo {
 struct storage_node {
 	struct list_head	all_link;
 	uint32_t		id;
+	bool			up;
 
 	unsigned		alen;
 	int			addr_af;
@@ -114,16 +115,22 @@ struct client_write {
 struct open_chunk {
 	struct st_client	*stc;
 	struct storage_node	*node;
+	struct list_head	link;
+	struct client		*cli;
 
 	uint64_t		wtogo;
 	uint64_t		wkey;
+	void (*wcb)(struct open_chunk *);
 	int			wfd;
+	bool			w_armed;
+	struct event		wevt;
+	size_t			wcnt;	/* in current buffer */
 
 	uint64_t		roff;
 	uint64_t		rsize;
 	void (*rcb)(struct open_chunk *);
 	int			rfd;
-	int			r_armed;
+	bool			r_armed;
 	struct event		revt;
 };
 
@@ -145,6 +152,7 @@ struct client {
 	struct sockaddr_in6	addr;		/* inet address */
 	char			addr_host[64];	/* ASCII version of inet addr */
 	int			fd;		/* socket */
+	bool			ev_active;
 	struct event		ev;
 	struct event		write_ev;
 
@@ -158,7 +166,7 @@ struct client {
 	char			*hdr_start;	/* current hdr start */
 	char			*hdr_end;	/* current hdr end (so far) */
 
-	struct open_chunk	out_ce;		/* just one for now FIXME */
+	struct list_head	out_ch;		/* open_chunk.link */
 	char			*out_bucket;
 	char			*out_key;
 	char			*out_user;
@@ -166,6 +174,9 @@ struct client {
 	long			out_len;
 	uint64_t		out_size;
 	uint64_t		out_objid;
+	char			*out_buf;
+	size_t			out_bcnt;	/* used length of out_buf */
+	int			out_nput;	/* number of users of out_buf */
 
 	struct open_chunk	in_ce;
 	unsigned char		*in_mem;
@@ -314,7 +325,9 @@ extern int stor_open_read(struct open_chunk *cep,
 			  uint64_t key, uint64_t *psz);
 extern void stor_close(struct open_chunk *cep);
 extern void stor_abort(struct open_chunk *cep);
-extern int stor_put_start(struct open_chunk *cep, uint64_t key, uint64_t size);
+extern int stor_put_start(struct open_chunk *cep,
+			  void (*cb)(struct open_chunk *),
+			  uint64_t key, uint64_t size);
 extern ssize_t stor_put_buf(struct open_chunk *cep, void *data, size_t len);
 extern bool stor_put_end(struct open_chunk *cep);
 extern ssize_t stor_get_buf(struct open_chunk *cep, void *data, size_t len);
server/tdbadm.c +5 -6
index e0e674e..823432a 100644
@@ -437,21 +437,20 @@ static void print_obj(struct db_obj_ent *obj)
 			GUINT16_FROM_LE(obj->size),
 			n_str);
 	} else {
-		printf("%s\t%s\t%s",
+		printf("%s\t%s\t%s\t%llX",
 			obj->bucket,
 			obj->owner,
-			obj->md5);
+			obj->md5,
+			(long long) GUINT64_FROM_LE(obj->d.a.oid));
 		for (i = 0; i < MAXWAY; i++) {
 			if (i == 0) {
 				printf("\t");
 			} else {
 				printf(",");
 			}
-			printf("\t%d:%lld",
-			       GUINT32_FROM_LE(obj->d.avec[i].nid),
-			       (long long) GUINT64_FROM_LE(obj->d.avec[i].oid));
-			printf("%u\n", n_str);
+			printf("%d", GUINT32_FROM_LE(obj->d.a.nidv[i]));
 		}
+		printf(" %u\n", n_str);
 	}
 
 	p = obj;
Applied
#1 lore raw
Quoted 11 lines · Pete Zaitcev

This is a small, but self-contained step toward the data redundancy. The patch adds ability to write data to several Chunk servers. Along the way, it fixes things up so we use asynchronous writes.

N.B. This patch changes the format of tabled's database in order to reflect that object IDs are global in the cell. This is done so the consistency checking and redundancy monitoring and recovery could be run without accessing the tabled's database. In orther words, it's a flag day, the database must be re-created.

applied