Skip to contentSkip to conversation

abstract out TCP-write code

PATCH Superseded Jeff Garzik 3 participants 11 messages Sep 23, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 abstract out TCP-write code

This is step #1. Other steps in the process:

2) update server/atcp.c and itd/*.c in tandem, until they have matching TCP-write code. should be straightforward, as both are based on the same codebase (tabled).
3) move atcp to libhail
4) remove TCP-write code from tabled, itd
5) update atcp to support SSL, sendfile
6) update chunkd to support atcp (req. step #5)

All this code bears the same lineage, so it shouldn't be too difficult.

Also note, this is a first draft with embedded libevent dependencies. I agree w/ zaitcev that the goal should be to eliminate these. atcp is wonderfully generic at present; not even a GLib dependency, IIRC.

 server/Makefile.am |    1 
 server/atcp.c      |  243 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 server/atcp.h      |   90 +++++++++++++++++++
 server/bucket.c    |    8 -
 server/object.c    |   56 ++++++------
 server/server.c    |  237 +++++----------------------------------------------
 server/tabled.h    |   37 +-------
 7 files changed, 400 insertions(+), 272 deletions(-)
server/Makefile.am +1 -0
index 5b53a0a..5e0abd5 100644
@@ -4,6 +4,7 @@ INCLUDES	= -I$(top_srcdir)/include @GLIB_CFLAGS@ @HAIL_CFLAGS@
 sbin_PROGRAMS	= tabled tdbadm
 
 tabled_SOURCES	= tabled.h		\
+		  atcp.c atcp.h		\
 		  bucket.c cldu.c config.c metarep.c object.c replica.c \
 		  server.c status.c storage.c storparse.c util.c
 tabled_LDADD	= ../lib/libtdb.a		\
server/atcp.c new file +243 -0

243 lines not on this page · raw message

server/atcp.h new file +90 -0

90 lines not on this page · raw message

server/bucket.c +4 -4
index eb03e03..982ed62 100644
@@ -566,13 +566,13 @@ bool bucket_add(struct client *cli, const char *user, const char *bucket)
 		     bucket) < 0)
 		return cli_err(cli, InternalError);
 
-	rc = cli_writeq(cli, hdr, strlen(hdr), cli_cb_free, hdr);
+	rc = atcp_writeq(&cli->wst, hdr, strlen(hdr), atcp_cb_free, cli, hdr);
 	if (rc) {
 		free(hdr);
 		return true;
 	}
 
-	return cli_write_start(cli);
+	return atcp_write_start(&cli->wst);
 
 err_out:
 	rc = txn->abort(txn);
@@ -718,13 +718,13 @@ bool bucket_del(struct client *cli, const char *user, const char *bucket)
 		     hutil_time2str(timestr, sizeof(timestr), time(NULL))) < 0)
 		return cli_err(cli, InternalError);
 
-	rc = cli_writeq(cli, hdr, strlen(hdr), cli_cb_free, hdr);
+	rc = atcp_writeq(&cli->wst, hdr, strlen(hdr), atcp_cb_free, cli, hdr);
 	if (rc) {
 		free(hdr);
 		return true;
 	}
 
-	return cli_write_start(cli);
+	return atcp_write_start(&cli->wst);
 
 err_out:
 	rc = txn->abort(txn);
server/object.c +30 -26

163 lines not on this page · raw message

server/server.c +25 -212

344 lines not on this page · raw message

server/tabled.h +7 -30
index d4d2048..be6c9b4 100644
@@ -31,6 +31,7 @@
 #include <elist.h>
 #include <tdb.h>
 #include <hail_log.h>
+#include "atcp.h"
 
 #ifndef ARRAY_SIZE
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
@@ -103,19 +104,6 @@ struct storage_node {
 };
 
 typedef bool (*cli_evt_func)(struct client *, unsigned int);
-typedef bool (*cli_write_func)(struct client *, void *, bool);
-
-struct client_write {
-	const void		*buf;		/* write buffer pointer */
-	int			togo;		/* write buffer remainder */
-
-	int			length;		/* length for accounting */
-	cli_write_func		cb;		/* callback */
-	void			*cb_data;	/* data passed to cb */
-	struct client		*cb_cli;	/* cli passed to cb */
-
-	struct list_head	node;
-};
 
 /* an open chunkd client */
 struct open_chunk {
@@ -163,13 +151,9 @@ struct client {
 	int			fd;		/* socket */
 	bool			ev_active;
 	struct event		ev;
-	struct event		write_ev;
 
-	struct list_head	write_q;	/* list of async writes */
-	size_t			write_cnt;	/* water level */
-	bool			writing;
+	struct atcp_wr_state	wst;
 	/* some debugging stats */
-	size_t			write_cnt_max;
 
 	unsigned int		req_used;	/* amount of req_buf in use */
 	char			*req_ptr;	/* start of unexamined data */
@@ -216,12 +200,12 @@ enum st_net {
 };
 
 struct server_stats {
-	unsigned long		poll;		/* number polls */
-	unsigned long		event;		/* events dispatched */
-	unsigned long		tcp_accept;	/* TCP accepted cxns */
-	unsigned long		opt_write;	/* optimistic writes */
+	uint64_t		poll;		/* number polls */
+	uint64_t		event;		/* events dispatched */
+	uint64_t		tcp_accept;	/* TCP accepted cxns */
+	uint64_t		opt_write;	/* optimistic writes */
 
-	unsigned long		max_write_buf;
+	uint64_t		max_write_buf;
 };
 
 #define DBID_NONE      0
@@ -249,7 +233,6 @@ struct server {
 	struct event_base	*evbase_main;
 	int			ev_pipe[2];
 	struct event		pevt;
-	struct list_head	write_compl_q;	/* list of done writes */
 	bool			mc_delay;
 	struct event		mc_timer;
 
@@ -398,12 +381,6 @@ extern bool cli_err(struct client *cli, enum errcode code);
 extern bool cli_err_write(struct client *cli, char *hdr, char *content);
 extern bool cli_resp_xml(struct client *cli, int http_status, GList *content);
 extern bool cli_resp_html(struct client *cli, int http_status, GList *content);
-extern int cli_writeq(struct client *cli, const void *buf, unsigned int buflen,
-		     cli_write_func cb, void *cb_data);
-extern size_t cli_wqueued(struct client *cli);
-extern bool cli_cb_free(struct client *cli, void *cb_data, bool done);
-extern bool cli_write_start(struct client *cli);
-extern bool cli_write_run_compl(void);
 extern int cli_req_avail(struct client *cli);
 extern void applog(int prio, const char *fmt, ...);
 extern void cld_update_cb(void);
#1 lore raw

server/server.c static void tcp_cli_event(int fd, short events, void *userdata) · line 1229

Quoted 8 lines · On Wed, 22 Sep 2010 20:09:08 -0400 Jeff Garzik <[email protected]> wrote:
server/server.c +1 -1
@@ -1410,7 +1224,7 @@ static void tcp_cli_event(int fd, short events, void *userdata)
 
 	do {
 		loop = cli->evt_table[cli->state](cli, events);
-		loop |= cli_write_run_compl();
+		loop |= atcp_write_run_compl(&cli->wst);
 	} while (loop);
 }

This cannot be right. Please see commit
d1a45fca7908b7128ed4fe2ab611111f02ee938f:

tabled: fix running completions over disposed cli

Miracluously this never actually crashed on me, but I added unrelated debugging printout into the dispatch routine and it printed weird values. Then it dawned on me that a state change function may dispose of the struct cli, in which case cli_write_run_compl is use-after-free.

It may seem that checking if the old state was evt_dispose before running cli_write_run_compl is an expedient fix, but that does not work, because we do not always dispose of the cli in such case. If the cli to be disposed still has anything in the queue, we need to continue to deliver events, and for that we have to run outstanding completions.

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

One other necessary change was to add a back poiter from a completion to the current client. This is because one caller needed the client pointer (object_get_more).

Speaking of backpointers, I think it would be much cleaner to get rid of two-argument format for callback. It stinks of special-casing. Either throw a back pointer into the first word of buf, or create some kind of object/struct passed as argument to atcp_writeq(), that's what I would do.

-- Pete

#2 lore raw
Quoted 35 lines · Pete Zaitcev

server/server.c static void tcp_cli_event(int fd, short events, void *userdata) · line 1229

Quoted 8 lines · On Wed, 22 Sep 2010 20:09:08 -0400 Jeff Garzik<[email protected]> wrote:
server/server.c +1 -1
@@ -1410,7 +1224,7 @@ static void tcp_cli_event(int fd, short events, void *userdata)
 
  	do {
  		loop = cli->evt_table[cli->state](cli, events);
-		loop |= cli_write_run_compl();
+		loop |= atcp_write_run_compl(&cli->wst);
  	} while (loop);
  }

This cannot be right. Please see commit
d1a45fca7908b7128ed4fe2ab611111f02ee938f:

tabled: fix running completions over disposed cli

Miracluously this never actually crashed on me, but I added unrelated debugging printout into the dispatch routine and it printed weird values. Then it dawned on me that a state change function may dispose of the struct cli, in which case cli_write_run_compl is use-after-free.

It may seem that checking if the old state was evt_dispose before running cli_write_run_compl is an expedient fix, but that does not work, because we do not always dispose of the cli in such case. If the cli to be disposed still has anything in the queue, we need to continue to deliver events, and for that we have to run outstanding completions.

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

Quoted 3 lines

One other necessary change was to add a back poiter from a completion to the current client. This is because one caller needed the client pointer (object_get_more).

Thanks for the reminder.

Looking at this change again, I don't see how this avoids use-after-free. If completions exist after state change function leads one to cli_evt_dispose() -> cli_free(), then cli_write_run_compl() still calls cli_write_free() with the stale 'cli' pointer.

It seems like the real fix is to have the functions in the FSM loop return an additional piece of information, indicating that 'cli' is no longer valid.

client_write's object lifetime should always be a subset of client's object lifetime.

Quoted 5 lines

Speaking of backpointers, I think it would be much cleaner to get rid of two-argument format for callback. It stinks of special-casing. Either throw a back pointer into the first word of buf, or create some kind of object/struct passed as argument to atcp_writeq(), that's what I would do.

It is a common idiom even in GLib that callbacks receive two anonymous pointers; witness the data type GFunc's 'data' and 'user_data' arguments: http://library.gnome.org/devel/glib/stable/glib-Doubly-Linked-Lists.html#GFunc

This is because typically one has two objects -- a parent and a child -- with different object lifetime. When considering both tabled and itd, you see example of this in the need for struct client (tabled) or struct session (itd) pointers on occasion, in cli_writeq calls.

Jeff

#3 lore raw
Quoted 10 lines · On Wed, 22 Sep 2010 21:26:13 -0400 Jeff Garzik <[email protected]> wrote:
Quoted 5 lines

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

Looking at this change again, I don't see how this avoids use-after-free. If completions exist after state change function leads one to cli_evt_dispose() -> cli_free(), then cli_write_run_compl() still calls cli_write_free() with the stale 'cli' pointer.

We run completions before freeing in all cases. My patch was correct.

Quoted 3 lines

It seems like the real fix is to have the functions in the FSM loop return an additional piece of information, indicating that 'cli' is no longer valid.

That's kinda backwards. Might as well add refcounts.

Quoted 2 lines

client_write's object lifetime should always be a subset of client's object lifetime.

Sure. But that's an argument for refcounting struct cli.

Quoted 10 lines
Quoted 5 lines

Speaking of backpointers, I think it would be much cleaner to get rid of two-argument format for callback. It stinks of special-casing. Either throw a back pointer into the first word of buf, or create some kind of object/struct passed as argument to atcp_writeq(), that's what I would do.

It is a common idiom even in GLib that callbacks receive two anonymous pointers; witness the data type GFunc's 'data' and 'user_data' arguments:
http://library.gnome.org/devel/glib/stable/glib-Doubly-Linked-Lists.html#GFunc

There's a lot of retarged garbage in Glib, just look at their lists. If someone smarter wrote Glib, we would not need struct list_head. Heck in fact queue.h existed before Glib and they still blew it. Oh god and their hash is no better.

Quoted 2 lines

This is because typically one has two objects -- a parent and a child -- with different object lifetime. []

I think your excuses are getting more sophisticated than the code is worth. If you don't see how 2 arguments are special-casing, fine. Just say you like it.

-- Pete

#4 lore raw
Quoted 9 lines · Pete Zaitcev
Quoted 5 lines · On Wed, 22 Sep 2010 21:26:13 -0400 Jeff Garzik<[email protected]> wrote:
Quoted 5 lines

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

Quoted 6 lines
Quoted 4 lines

Looking at this change again, I don't see how this avoids use-after-free. If completions exist after state change function leads one to cli_evt_dispose() -> cli_free(), then cli_write_run_compl() still calls cli_write_free() with the stale 'cli' pointer.

We run completions before freeing in all cases. My patch was correct.

Logically, if completions are run before freeing in all cases, there is no need to make write_compl_q global. That was a red herring, which by side effect avoided the bug with the stale 'cli' pointer.

Jeff

#5 lore raw
Quoted 11 lines · On Thu, 23 Sep 2010 00:32:09 -0400 Jeff Garzik <[email protected]> wrote:
Quoted 11 lines
Quoted 10 lines
Quoted 5 lines

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

Looking at this change again, I don't see how this avoids use-after-free. If completions exist after state change function leads one to cli_evt_dispose() -> cli_free(), then cli_write_run_compl() still calls cli_write_free() with the stale 'cli' pointer.

Quoted 5 lines
Quoted 1 line

We run completions before freeing in all cases. My patch was correct.

Logically, if completions are run before freeing in all cases, there is no need to make write_compl_q global. That was a red herring, which by side effect avoided the bug with the stale 'cli' pointer.

Side effect or not, if one applies your patch and executes "export MALLOC_PERTURB_=43" command before "make check", the result is a crash:

Core was generated by `../server/tabled -C ../test/tabled-test.conf -E'. Program terminated with signal 11, Segmentation fault.
#0 atcp_write_free (tmp=0x2b2b2b2b2b2b2afb, done=true) at atcp.c:49
49 struct atcp_wr_state *wst = tmp->wst;
(gdb) where

#0  atcp_write_free (tmp=0x2b2b2b2b2b2b2afb, done=true) at atcp.c:49
#1  0x000000000040387e in atcp_write_run_compl (wst=0x15a9be8) at atcp.c:70
#2  0x000000000040f20a in tcp_cli_event (fd=<value optimized out>, events=2,
    userdata=0x15a9af0) at server.c:1227

#3 0x00007f9320aec774 in event_base_loop () from /usr/lib64/libevent-1.4.so.2 #4 0x00000000004107a5 in main (argc=<value optimized out>,
argv=<value optimized out>) at server.c:2139
(gdb)

The existing code (with the commit that you criticized) produces no crash. Granted MALLOC_PERTURB_ is like SLAB debug option -- is not the normal operating environment -- but IMHO it is completely legitimate.

-- Pete

#6 lore raw
Quoted 33 lines · Pete Zaitcev
Quoted 11 lines · On Thu, 23 Sep 2010 00:32:09 -0400 Jeff Garzik <[email protected]> wrote:
Quoted 11 lines
Quoted 10 lines
Quoted 5 lines

So, we go a longer route and re-hook the list of completions to a per-server global instead of a client. The patch is straight- forward. The only thing we need to be careful is to make sure that no outstanding completions are left in the queue before freeing a client struct. This is ensured by force-running completions.

Looking at this change again, I don't see how this avoids use-after-free. If completions exist after state change function leads one to cli_evt_dispose() -> cli_free(), then cli_write_run_compl() still calls cli_write_free() with the stale 'cli' pointer.

Quoted 5 lines
Quoted 1 line

We run completions before freeing in all cases. My patch was correct.

Logically, if completions are run before freeing in all cases, there is no need to make write_compl_q global. That was a red herring, which by side effect avoided the bug with the stale 'cli' pointer.

Side effect or not, if one applies your patch and executes "export MALLOC_PERTURB_=43" command before "make check", the result is a crash:

Core was generated by `../server/tabled -C ../test/tabled-test.conf -E'. Program terminated with signal 11, Segmentation fault.
#0 atcp_write_free (tmp=0x2b2b2b2b2b2b2afb, done=true) at atcp.c:49
49 struct atcp_wr_state *wst = tmp->wst;
(gdb) where

#0  atcp_write_free (tmp=0x2b2b2b2b2b2b2afb, done=true) at atcp.c:49
#1  0x000000000040387e in atcp_write_run_compl (wst=0x15a9be8) at atcp.c:70
#2  0x000000000040f20a in tcp_cli_event (fd=<value optimized out>, events=2,
    userdata=0x15a9af0) at server.c:1227
Quoted 8 lines

#3 0x00007f9320aec774 in event_base_loop () from /usr/lib64/libevent-1.4.so.2 #4 0x00000000004107a5 in main (argc=<value optimized out>,
argv=<value optimized out>) at server.c:2139
(gdb)

The existing code (with the commit that you criticized) produces no crash. Granted MALLOC_PERTURB_ is like SLAB debug option -- is not the normal operating environment -- but IMHO it is completely legitimate.

Every developer should have MALLOC_PERTURB_=N (N in 1..255) set in his/her environment on glibc-based systems. Almost all the time.

Maybe I should push harder to get it added to rawhide's /etc/profile. I proposed that a few months ago:

use MALLOC_PERTURB_ ... or lose
http://thread.gmane.org/gmane.linux.redhat.fedora.devel/132690

#7 lore raw
Quoted 2 lines · Jim Meyering

Every developer should have MALLOC_PERTURB_=N (N in 1..255) set in his/her environment on glibc-based systems. Almost all the time.

I heard about it a while ago, even submitted a bugzilla bug to have it documented adequately. But apparently its absent from my .bash_profile.
Added.

Jeff

#8 lore raw
In reply to Pete Zaitcev
Quoted 3 lines · Pete Zaitcev

Side effect or not, if one applies your patch and executes "export MALLOC_PERTURB_=43" command before "make check", the result is a crash:

Yes, it is clear my patch re-introduces the problem. I thanked you for pointing that out in the original reply. :)

The point was, that does not invalidate the approach of storing write_compl_q in struct atcp_wr_state.

It merely highlights that a further fix (to the FSM? refcount client? etc.) is required.

Jeff

#9 lore raw
In reply to Pete Zaitcev
Quoted 9 lines · Pete Zaitcev
Quoted 4 lines · On Wed, 22 Sep 2010 21:26:13 -0400 Jeff Garzik<[email protected]> wrote:

It is a common idiom even in GLib that callbacks receive two anonymous pointers; witness the data type GFunc's 'data' and 'user_data' arguments:
http://library.gnome.org/devel/glib/stable/glib-Doubly-Linked-Lists.html#GFunc

There's a lot of retarged garbage in Glib, just look at their lists. If someone smarter wrote Glib, we would not need struct list_head.

I use both list types, because there's a use case for both. You don't always have the luxury of having a struct in which to embed data+next pointers. Allocated strings are an excellent example.

GFunc has two parameters for a reason :) See for example http://library.gnome.org/devel/glib/stable/glib-Doubly-Linked-Lists.html#g-list-foreach

It really is a common idiom, based on a common need, not just my style preference. :)

Jeff

#10 lore raw

PATCH v2 1/1 abstract out TCP-write code

In reply to Jeff Garzik

Changes from v1:
- avoid referencing dead struct client (grep for 'invalidate_cli'), by changing FSM callback prototype.
- insert 'void *priv' member into struct atcp_wr_state, and replace cb_data1/cb_data2 callback parameters with (struct atcp_wr_state *, void *). struct client / struct session, or whatever, may be stored in atcp_wr_state::priv.
- minor API polishing and further abstraction

 server/Makefile.am |    1 
 server/atcp.c      |  238 +++++++++++++++++++++++++++++++++++++++++++++++
 server/atcp.h      |  100 +++++++++++++++++++
 server/bucket.c    |    8 -
 server/object.c    |   56 +++++------
 server/server.c    |  268 +++++++++--------------------------------------------
 server/status.c    |    3 
 server/tabled.h    |   46 ++-------
 8 files changed, 436 insertions(+), 284 deletions(-)
server/Makefile.am +1 -0
index 5b53a0a..5e0abd5 100644
@@ -4,6 +4,7 @@ INCLUDES	= -I$(top_srcdir)/include @GLIB_CFLAGS@ @HAIL_CFLAGS@
 sbin_PROGRAMS	= tabled tdbadm
 
 tabled_SOURCES	= tabled.h		\
+		  atcp.c atcp.h		\
 		  bucket.c cldu.c config.c metarep.c object.c replica.c \
 		  server.c status.c storage.c storparse.c util.c
 tabled_LDADD	= ../lib/libtdb.a		\
server/atcp.c new file +238 -0

238 lines not on this page · raw message

server/atcp.h new file +100 -0

100 lines not on this page · raw message

server/bucket.c +4 -4
index eb03e03..a81eca1 100644
@@ -566,13 +566,13 @@ bool bucket_add(struct client *cli, const char *user, const char *bucket)
 		     bucket) < 0)
 		return cli_err(cli, InternalError);
 
-	rc = cli_writeq(cli, hdr, strlen(hdr), cli_cb_free, hdr);
+	rc = atcp_writeq(&cli->wst, hdr, strlen(hdr), atcp_cb_free, hdr);
 	if (rc) {
 		free(hdr);
 		return true;
 	}
 
-	return cli_write_start(cli);
+	return atcp_write_start(&cli->wst);
 
 err_out:
 	rc = txn->abort(txn);
@@ -718,13 +718,13 @@ bool bucket_del(struct client *cli, const char *user, const char *bucket)
 		     hutil_time2str(timestr, sizeof(timestr), time(NULL))) < 0)
 		return cli_err(cli, InternalError);
 
-	rc = cli_writeq(cli, hdr, strlen(hdr), cli_cb_free, hdr);
+	rc = atcp_writeq(&cli->wst, hdr, strlen(hdr), atcp_cb_free, hdr);
 	if (rc) {
 		free(hdr);
 		return true;
 	}
 
-	return cli_write_start(cli);
+	return atcp_write_start(&cli->wst);
 
 err_out:
 	rc = txn->abort(txn);
server/object.c +30 -26

167 lines not on this page · raw message

server/server.c +48 -220

414 lines not on this page · raw message

server/status.c +2 -1
index e9fbb38..9af60f7 100644
@@ -150,7 +150,8 @@ out_err:
 	return cli_err(cli, InternalError);
 }
 
-bool stat_evt_http_req(struct client *cli, unsigned int events)
+bool stat_evt_http_req(struct client *cli, unsigned int events,
+			bool *invalidate_cli)
 {
 	struct http_req *req = &cli->req;
 	char *method = req->method;
server/tabled.h +13 -33

97 lines not on this page · raw message