PROBLEM: select() on TCP socket sleeps for 1 tick even if data available

Kernel mods are less drastic than a system call? :^)

Reading the documentation for sched_setscheduler, it's unclear whether it would even suffice - it seems to deal with preempting other processes. I am not trying to get into the run queue ahead of other processes, the machine is sitting there IDLE and my process is still not getting to run for a full clock tick.

You know, there's one other possibility, and that's if the data that is being sent isn't actually arriving until the next clock tick, which means the delay is in the appearance of sent data, not in select(). Given that the two processes are on the same machine, I would expect a send() on a TCP socket to deliver the data to its destination faster than that, however.

--
Mike Lindner
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

On Sat, Jan 20, 2001 at 01:54:23AM -0500, Michael Lindner wrote:

You know, there's one other possibility, and that's if the data that is being sent isn't actually arriving until the next clock tick, which means the delay is in the appearance of sent data, not in select(). Given that the two processes are on the same machine, I would expect a send() on a TCP socket to deliver the data to its destination faster than that, however.

You can measure this latency; and it's indeed very low (lmbench gives 28 usecs on one of my machines).

If process A blocks waiting for data, and process B sleeps after writing this data intended to wake process A, it should wake almost immediately.

If you don't see this I would suspect an application bug -- can you use strace or some such and confirm this is not the case?

--cw
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

OK, two new data points (thanks for staying with me here):

1. The problem only occurs when traffic is travelling over DIFFERENT sockets (i.e. A->B->C->D... or A->B->A but using a separate socket for traffic in each direction).

2. I wrote a very ugly program (attached) to reproduce the problem. Lest you think ill of me, most of this isn't actual code I wrote (the actual program that first reproduced the problem was in C++). Just run

	sockperf localhost 54321 54322
	sockperf localhost 54322 54321 1

to see it in action.

--
Mike Lindner

#include <fcntl.h>
#include <memory.h>
#include <netdb.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdio.h>
#include <sys/select.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <varargs.h>
#include <netinet/in.h>
#include <netdb.h>
#include <errno.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <unistd.h>

#ifndef INADDR_NONE
#define INADDR_NONE ~0
#endif

void
errexit(format, va_alist)
char *format;
va_dcl

{
	va_list	args;
	va_start(args);
	vfprintf(stderr, format, args);
	va_end(args);
	exit(1);
}

/*
* passivesock - allocate & bind a server socket using TCP or UDP */

int
passivesock( service, protocol, qlen )
char *service; /* service associeted with the desired port */
char *protocol; /* name of protocol to use ("tcp" or "udp") */
int qlen; /* maximum length of the server request queue */

{
	struct servent *pse;
	struct protoent *ppe;
	struct sockaddr_in sin;
	int	s, type;
	int	one = 1;

	bzero((char *) & sin, sizeof(sin));
	sin.sin_family = AF_INET;
	sin.sin_addr.s_addr = INADDR_ANY;

	/* Map service name to port number */
	if ( pse = getservbyname(service, protocol) )
		sin.sin_port = htons(ntohs((u_short)pse->s_port));
	else if ( (sin.sin_port = htons((u_short)atoi(service))) == 0 )
		errexit("can't get \"%s\" service entry\n", service);

	/* Map protocol name to protocol number */
	if ( (ppe = getprotobyname(protocol)) == 0)
		errexit("can't get \"%s\" protocol entry\n", protocol);

	/* Use protocol to chose a socket type */
	if (strcmp(protocol, "udp") == 0)
		type = SOCK_DGRAM;
	else
		type = SOCK_STREAM;

	/* Allocate a socket */
	s = socket(PF_INET, type, ppe->p_proto);
	if (s < 0 )
		errexit("can't create socket: %s\n", strerror(errno));

	setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
	/* Bind the socket */
	if (bind(s, (struct sockaddr *) & sin, sizeof(sin)) < 0)
		errexit("can't bind to %s port: %s\n", service,
		    strerror(errno));
	if (type == SOCK_STREAM && listen(s, qlen) < 0)
		errexit("can't listen on %s port: %s\n", service,
		    strerror(errno));
	return s;
}

int
connectsock(host, service, protocol)
char *host;
char *service;
char *protocol;

{
	struct hostent	*phe;
	struct servent	*pse;
	struct protoent	*ppe;
	struct sockaddr_in	sin;
	int	s, type;

	memset(&sin, 0, sizeof(sin));
	if (pse = getservbyname(service, protocol))
		sin.sin_port = pse->s_port;
	else if ((sin.sin_port = htons((u_short) atoi(service))) == 0) {
		fprintf(stderr, "can't get '%s' service entry\n", service);
		exit(1);
	}
	if (phe = gethostbyname(host)) 
		memcpy((char *) &sin.sin_addr, phe->h_addr, phe->h_length);
	else if ((sin.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) {
		fprintf(stderr, "can't get '%s' host entry\n", host);
		exit(1);
	}
	/* if (ppe = getprotobyname(protocol)) {
		fprintf(stderr, "can't get '%s' protocol entry\n", protocol);
		exit(1);
	}
	if (strcmp(protocol, "udp") == 0)
		type = SOCK_DGRAM;
	else
		type = SOCK_STREAM;
	*/
	sin.sin_family = AF_INET;
	s = socket(AF_INET, SOCK_STREAM, 6);
	if (s < 0) {
		perror("can't create socket\n");
		exit(1);
	}
	if (connect(s, (struct sockaddr *) &sin, sizeof(sin)) < 0) {
		perror("can't connect to socket");
		exit(1);
	}
	return s;
}

void
pingpong(int r, int s, int ping)

{
	struct timeval then;
	struct timeval now;
	fd_set fds;
	fd_set readfds;
	int pings = 0;
	FD_ZERO(&fds);
	FD_SET(r, &fds);
	gettimeofday(&then, 0);
	if (ping) {
		send(s, ".", 1, 0);
		pings++;
	}
	readfds = fds;
	while (select(r+1, &readfds, 0, 0, 0) > 0) {
		if (FD_ISSET(r, &readfds)) {
			char buf[1];
			int n = read(r, buf, sizeof(buf));
			if (n <= 0) {
				break;
			}
			else {
				if (pings++ < 1000) {
					send(s, ".", 1, 0);
				}
				else {
					break;
				}
			}
		}
		else {
			fprintf(stderr, "fd not set!\n");
		}
		readfds = fds;
	}
	gettimeofday(&now, 0);
	fprintf(stderr, "elapsed time for 1000 pingpongs is %g\n", now.tv_sec - then.tv_sec + (now.tv_usec - then.tv_usec) / 1000000.0);
	fprintf(stderr, "closing %d\n", r);
	close(r);
	fprintf(stderr, "closing %d\n", s);
	close(s);
}

main(argc, argv)
int argc;
char **argv;

{
	char	buf[1024];
	int	n;
	int	s;
	int	f;
	if (argc < 3) {
		errexit("usage: %s host port1 port2 [initiate]\n", argv[0]);
	}
	signal(SIGPIPE, SIG_IGN);
	f = passivesock(argv[2], "tcp", 2);
	if (f < 0) {
		errexit("listen failed: %s\n", strerror(errno));
	}
	for ( ; ; ) {
		struct sockaddr_in	fsin;
		int	alen = sizeof(fsin);
		if (argc < 5) {
			int	r = accept(f, (struct sockaddr *) &fsin, &alen);
			int	s = connectsock(argv[1], argv[3], "tcp");
			if (r < 0)
				errexit("accept failed: %s\n", strerror(errno));
			if (s < 0)
				errexit("connect failed: %s\n", strerror(errno));
			pingpong(r, s, 0);
		}
		else {
			int	s = connectsock(argv[1], argv[3], "tcp");
			int	r = accept(f, (struct sockaddr *) &fsin, &alen);
			if (r < 0)
				errexit("accept failed: %s\n", strerror(errno));
			if (s < 0)
				errexit("connect failed: %s\n", strerror(errno));
			pingpong(r, s, 1);
			break;
		}
	}
	return 0;
}

What kernel have you been using? I have reproduced your problem on a standard 2.2.18 kernel (elapsed time ~10sec). However, using a 2.4.0 kernel with HZ=1000, I see a 100x improvement (elapsed time ~0.1 sec; note that increasing HZ alone should only give a 10x improvement). Perhaps the scheduler was fixed in 2.4.0?

2.2.18 very definitely has some scheduling anomalies. In your benchmark, select() or poll() takes 10ms, as can be observed with strace -T. Skipping the select() and blocking in read() gives the same behavior. This leads me to believe the scheduler is at fault, and not select(), poll(), or read().

When run without strace, 2.4.0 appears to have no problems with your benchmark. Elapsed time is 0.1 sec -- this may be the full potential of my machine (PII/450). Removing select() and blocking in read() results in a further improvement, to 0.07 sec.

Strace disturbs the behavior of 2.4.0 in strange ways. Running the benchmark under strace with 2.4.0 causes the scheduler delays to return -- ~1ms delays appear in select() or write(). This is confusing - it appears that context switches can happen inside write() as well as select(), a result I don't understand at all (the socket buffers never completely fill since you only write 1000 bytes to each one).

Other notes: poll() behaves same as select(). Using the SCHED_FIFO class and mlockall() has no effect on this benchmark. Setting the sockets non-blocking also has no effect.

I wish I had the Linux Trace Toolkit handy; it would give a much better idea of what's going on than strace...

Dan

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Sounds like a good reason for me to upgrade - I am running 2.2.18 now. If it's fixed in 2.4.0, then I'm happy (although I'm usually leery of installing ANYTHING that ends in ".0", Linux has never been anything less than stable). It sounds like there are some other anomalies this tickles that might bear looing into, though.

Thanks for all the help, and again, my apologies for posting a lame-o test program with the original report - had I taken the time to make a test program that REALLY exercised the problem as described, I would have saved you all a lot of time.

--
Mike Lindner
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Ever heard of nagle? (If not, there's a long thread about it on the mailing list *g*)

It's not the select that waits. It's a delay in the tcp send path waiting for more data. Try disabling it:

	int f=1;
	setsockopt(s, SOL_TCP, TCP_NODELAY, &f, sizeof(f));

Ciao, ET.

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Bingo! With this fix, 2.2.18 performance becomes almost identical to 2.4.0 performance. I assume 2.4.0 disables Nagle by default on local connections...

Dan

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

On Sat, Jan 20, 2001 at 07:35:12PM -0500, Dan Maas wrote:

Bingo! With this fix, 2.2.18 performance becomes almost identical to 2.4.0 performance. I assume 2.4.0 disables Nagle by default on local connections...

2.4.x has a smarter nagle algorithm.

--cw

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

can someone explain what is nagle or pinpoint explanation :)

lynx

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

Thanks again for all the help, guys...

Haven't installed 2.4 yet, but I tried the setsockoption route. Performance is better, but the two processes together never total more than 50% of the CPU (i.e. the thing is still schedule-bound, not compute bound, as it is on other platforms), and throughput is only up to 800 sends/sec. Better than the 100/sec. I was getting, but still a far cry from the identical box running Windows, where performance is 8K/sec.

...and I still don't understand why the identical program, but using one socket instead of 2 sockets, IS CPU bound, and gets on the order of 10K/sec. on the same HW. Diffs to produce 10K/sec. 1 socket version from my previous sample follow...

--
Mike Lindner

diff sockperf.c sockperf1.c
163c163
< if (pings++ < 1000) {
---

177c177
< fprintf(stderr, "elapsed time for 1000 pingpongs is %g\n",
now.tv_sec - then.tv_sec + (now.tv_usec - then.tv_usec) / 1000000.0);
---

205c205
< int s = connectsock(argv[1], argv[3],
"tcp");
---

214c214
< int r = accept(f, (struct sockaddr *) &fsin,
&alen);
---

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

It's really this simple -- this isn't what TCP is intended for.

DS

-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

OK, 2.4.0 kernel installed, and a new set of numbers:

test kernel ping-pongs/s. @ total CPU util w/SOL_NDELAY
sample (2 skts) 2.2.18 100 @ 0.1% 800 @ 1%
sample (1 skt) 2.2.18 8000 @ 100% 8000 @ 50%
real app 2.2.18 100 @ 0.1% 800 @ 1%

sample (2 skts) 2.4.0 8000 @ 50% 8000 @ 50%
sample (1 skt) 2.4.0 10000 @ 50% 10000 @ 50%
real app 2.4.0 1200 @ 50% 1200 @ 50%

real app Windows 2K 4000 @ 100%

The two points that still seem strange to me are:

1. The 1 socket case is still 25% faster than the 2 socket case in 2.4.0 (in 2.2.18 the 1 socket case was 10x faster).

2. Linux never devotes more than 50% of the CPU (average over a long run) to the two processes (25% to each process, with the rest of the time idle).

I'd really love to show that Linux is a viable platform for our SW, and I think it would be doable if I could figure out how to get the other 50% of my CPU involved. An "strace -rT" of the real app on 2.4.0 looks like this for each ping/pong.

     0.052371 send(7, "\0\0\0
\177\0\0\1\3243\0\0\0\2\4\236\216\341\0\0\v\277"..., 32, 0) = 32
<0.000529>
     0.000882 rt_sigprocmask(SIG_BLOCK, ~[], [RT_0], 8) = 0 <0.000021>
     0.000242 rt_sigprocmask(SIG_SETMASK, [RT_0], NULL, 8) = 0
<0.000021>
     0.000173 select(8, [3 4 6 7], NULL, NULL, NULL) = 1 (in [6])
<0.000047>
     0.000328 read(6, "\0\0\0 ", 4)     = 4 <0.000031>
     0.000179 read(6,

"\177\0\0\1\3242\0\0\0\2\4\236\216\341\0\0\7\327\177\0\0"..., 28) = 28 <0.000075>

--
Mike Lindner
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/

An old thread, but important to get these fundamental performance numbers up there:

2.4.2 on an 800mhz PIII Sceptre laptop w/ 512MB ram:

elapsed time for 100000 pingpongs is
3.81327
100000/3.81256

        ~26229.09541095746689888159     
10000/.379912
        ~26321.88506812103855629724  

26300 compares to 8000/sec. quite well ;-) You didn't give specs for your test machine unfortunately.

Since this tests both 'sides' of an application communication, it indicates a 'null transaction' rate of twice that.

This was typical cpu usage on a triple run of 10000:
CPU states: 7.2% user, 92.7% system, 0.0% nice, 0.0% idle

sdw

[email protected]  http://sdw.st
Stephen D. Williams
43392 Wayside Cir,Ashburn,VA 20147-4622 703-724-0118W 703-995-0407Fax 
Dec2000

I seemed to miss the original post, so I can't really comment on the
tests. However...

The strace here shows select() with an infinite timeout, you're
numbers will be much better if you do (pseudo code)...

struct timeval zerotime;

  zerotime.tv_sec = 0;
  zerotime.tv_usec = 0;

if (!(ret = select( ... , &zerotime)))
ret = select( ... , NULL);

...basically you completely miss the function call for __pollwait() inside poll_wait (include/linux/poll.h in the linux sources, with __pollwait being in fs/select.c).

# James Antill -- [email protected]
:0:
* ^From: .*james@and\.org
/dev/null

...

It was a thread in January, but just ran accross it looking for something else. See below for results.

...

Apparently the extra system call overhead outweighs any benefit. In any case, what you suggest would be better done in the kernel anyway. The time went from 3.7 to 4.4 seconds per 100000.

[email protected]  http://sdw.st
Stephen D. Williams
43392 Wayside Cir,Ashburn,VA 20147-4622 703-724-0118W 703-995-0407Fax 
Dec2000

Ahh, ok.

[snip ... ]

There shouldn't be any "extra" system calls in the fast path. If data
is waiting then you do one call to poll() either way, if not then you are wasting time blocking so it doesn't matter what you do.

Possibly, however when this has come up before the kernel people have
said it's hard to do in kernel space.

Ok here's a quick test that I've done. This passes data between 2
processes. Obviously you can't compare this to your code or Michael's, however...

The results with USE_DOUBLE_POLL on are...

% time ./pingpong
./pingpong 0.15s user 0.89s system 48% cpu 2.147 total % time ./pingpong
./pingpong 0.19s user 0.91s system 45% cpu 2.422 total % time ./pingpong
./pingpong 0.10s user 1.02s system 49% cpu 2.282 total

The results with USE_DOUBLE_POLL off are...

% time ./pingpong
./pingpong 0.24s user 1.07s system 50% cpu 2.614 total % time ./pingpong
./pingpong 0.21s user 1.00s system 44% cpu 2.695 total % time ./pingpong
./pingpong 0.21s user 1.13s system 50% cpu 2.667 total

Don't forget that the poll here is done with _1_ fd. Most real
programs have more, and so benifit more.

I also did the TRY_NO_POLL, as I was pretty sure what the results
would be, that gives...

% time ./pingpong
./pingpong 0.03s user 0.41s system 50% cpu 0.874 total % time ./pingpong
./pingpong 0.06s user 0.44s system 58% cpu 0.855 total % time ./pingpong
./pingpong 0.07s user 0.35s system 51% cpu 0.820 total

-- 
# James Antill -- [email protected]
:0:
* ^From: .*james@and\.org
/dev/null

James Antill wrote:
...

I've attached my version of his code with your suggested change. Possibly I didn't do it correctly.

...

sdw

[email protected]  http://sdw.st
Stephen D. Williams
43392 Wayside Cir,Ashburn,VA 20147-4622 703-724-0118W 703-995-0407Fax 
Dec2000

#include <fcntl.h>
#include <memory.h>
#include <netdb.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdio.h>
#include <sys/select.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <varargs.h>
#include <netinet/in.h>
#include <netdb.h>
#include <errno.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <unistd.h>
#include <netinet/tcp.h>

#ifndef INADDR_NONE
#define INADDR_NONE ~0
#endif

void
errexit(format, va_alist)
char *format;
va_dcl

{
        va_list args;
        va_start(args);
        vfprintf(stderr, format, args);
        va_end(args);
        exit(1);
}

/*
* passivesock - allocate & bind a server socket using TCP or UDP */

int
passivesock( service, protocol, qlen )

char    *service;       /* service associeted with the desired port     */
char    *protocol;      /* name of protocol to use ("tcp" or "udp")     */
int     qlen;           /* maximum length of the server request queue   */
{
        struct servent *pse;
        struct protoent *ppe;
        struct sockaddr_in sin;
        int     s, type;
        int     one = 1;
        int f=1;

        bzero((char *) & sin, sizeof(sin));
        sin.sin_family = AF_INET;
        sin.sin_addr.s_addr = INADDR_ANY;

        /* Map service name to port number */
        if ( pse = getservbyname(service, protocol) )
                sin.sin_port = htons(ntohs((u_short)pse->s_port));
        else if ( (sin.sin_port = htons((u_short)atoi(service))) == 0 )
                errexit("can't get \"%s\" service entry\n", service);

        /* Map protocol name to protocol number */
        if ( (ppe = getprotobyname(protocol)) == 0)
                errexit("can't get \"%s\" protocol entry\n", protocol);

        /* Use protocol to chose a socket type */
        if (strcmp(protocol, "udp") == 0)
                type = SOCK_DGRAM;
        else
                type = SOCK_STREAM;

        /* Allocate a socket */
        s = socket(PF_INET, type, ppe->p_proto);
        if (s < 0 )
                errexit("can't create socket: %s\n", strerror(errno));

        setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
        setsockopt(s, SOL_TCP, TCP_NODELAY, &f, sizeof(f));
        /* Bind the socket */
        if (bind(s, (struct sockaddr *) & sin, sizeof(sin)) < 0)
                errexit("can't bind to %s port: %s\n", service,
                    strerror(errno));
        if (type == SOCK_STREAM && listen(s, qlen) < 0)
                errexit("can't listen on %s port: %s\n", service,
                    strerror(errno));
        return s;
}

int
connectsock(host, service, protocol)

char    *host;
char    *service;
char    *protocol;
{
        struct hostent  *phe;
        struct servent  *pse;
        struct protoent *ppe;
        struct sockaddr_in      sin;
        int     s, type;
        int f=1;

        memset(&sin, 0, sizeof(sin));
        if (pse = getservbyname(service, protocol))
                sin.sin_port = pse->s_port;
        else if ((sin.sin_port = htons((u_short) atoi(service))) == 0) {
                fprintf(stderr, "can't get '%s' service entry\n", service);
                exit(1);
        }
        if (phe = gethostbyname(host)) 
                memcpy((char *) &sin.sin_addr, phe->h_addr, phe->h_length);
        else if ((sin.sin_addr.s_addr = inet_addr(host)) == INADDR_NONE) {
                fprintf(stderr, "can't get '%s' host entry\n", host);
                exit(1);
        }
        /* if (ppe = getprotobyname(protocol)) {
                fprintf(stderr, "can't get '%s' protocol entry\n", protocol);
                exit(1);
        }
        if (strcmp(protocol, "udp") == 0)
                type = SOCK_DGRAM;
        else
                type = SOCK_STREAM;
        */
        sin.sin_family = AF_INET;
        s = socket(AF_INET, SOCK_STREAM, 6);
        setsockopt(s, SOL_TCP, TCP_NODELAY, &f, sizeof(f));
        if (s < 0) {
                perror("can't create socket\n");
                exit(1);
        }
        if (connect(s, (struct sockaddr *) &sin, sizeof(sin)) < 0) {
                perror("can't connect to socket");
                exit(1);
        }
        return s;
}

void
pingpong(int r, int s, int ping)

{
        struct timeval then;
        struct timeval now;
        fd_set fds;
        fd_set readfds;
        int pings = 0;
	struct timeval zerotime;
	int ret;

        FD_ZERO(&fds);
        FD_SET(r, &fds);
        gettimeofday(&then, 0);
        if (ping) {
                send(s, ".", 1, 0);
                pings++;
        }
        readfds = fds;

	zerotime.tv_sec = 0;
	zerotime.tv_usec = 0;

	//	if (!(ret = select( ... , &zerotime)))
	//  ret = select( ... , NULL);
	//        while ((readfds=fds, ret = select(r+1, &readfds, 0, 0, 0)) ) {
	while ((ret = select(r+1, &readfds, 0, 0, &zerotime)) ||
	       (readfds=fds, ret = select(r+1, &readfds, 0, 0, 0)) ) {
	       if (FD_ISSET(r, &readfds)) {
                        char buf[1];
                        int n = read(r, buf, sizeof(buf));
                        if (n <= 0) {
                                break;
                        }
                        else {
                                if (pings++ < 100000) {
                                        send(s, ".", 1, 0);
                                }
                                else {
                                        break;
                                }
                        }
                }
                else {
                        fprintf(stderr, "fd not set!\n");
                }
                readfds = fds;
        }
        gettimeofday(&now, 0);
        fprintf(stderr, "elapsed time for 100000 pingpongs is %g\n", now.tv_sec - then.tv_sec + (now.tv_usec -
then.tv_usec) / 1000000.0);
        fprintf(stderr, "closing %d\n", r);
        close(r);
        fprintf(stderr, "closing %d\n", s);
        close(s);
}

main(argc, argv)
int argc;
char **argv;

{
        char    buf[1024];
        int     n;
        int     s;
        int     f;
        if (argc < 3) {
                errexit("usage: %s host port1 port2 [initiate]\n", argv[0]);
        }
        signal(SIGPIPE, SIG_IGN);
        f = passivesock(argv[2], "tcp", 2);
        if (f < 0) {
                errexit("listen failed: %s\n", strerror(errno));
        }
        for ( ; ; ) {
                struct sockaddr_in      fsin;
                int     alen = sizeof(fsin);
                if (argc < 5) {
                        int     r = accept(f, (struct sockaddr *) &fsin, &alen);
                        int     s = connectsock(argv[1], argv[3], "tcp");
                        if (r < 0)
                                errexit("accept failed: %s\n", strerror(errno));
                        if (s < 0)
                                errexit("connect failed: %s\n", strerror(errno));
                        pingpong(r, s, 0);
                }
                else {
                        int     s = connectsock(argv[1], argv[3], "tcp");
                        int     r = accept(f, (struct sockaddr *) &fsin, &alen);
                        if (r < 0)
                                errexit("accept failed: %s\n", strerror(errno));
                        if (s < 0)
                                errexit("connect failed: %s\n", strerror(errno));
                        pingpong(r, s, 1);
                        break;
                }
        }
        return 0;
}

It's not a code thing, but I think you are measuring the wrong thing
(at least in relation to the original question). Given the below diff...

@@ -17,6 +17,7 @@
 #include <unistd.h>
 #include <netinet/tcp.h>
 
+#define USE_DOUBLE_SELECT 0
 
 #ifndef INADDR_NONE
 #define INADDR_NONE     ~0

@@ -147,7 +148,8 @@
         int pings = 0;
 	struct timeval zerotime;
 	int ret;
-
+        unsigned int misses = 0;
+        
         FD_ZERO(&fds);
         FD_SET(r, &fds);
         gettimeofday(&then, 0);

@@ -163,8 +165,10 @@
 	//	if (!(ret = select( ... , &zerotime)))
 	//  ret = select( ... , NULL);
 	//        while ((readfds=fds, ret = select(r+1, &readfds, 0, 0, 0)) ) {-	while ((ret = select(r+1, &readfds, 0, 0, &zerotime)) ||
-	       (readfds=fds, ret = select(r+1, &readfds, 0, 0, 0)) ) {
+	while ((USE_DOUBLE_SELECT &&
+                (ret = select(r+1, &readfds, 0, 0, &zerotime))) ||
+	       (++misses &&
+                (readfds=fds, ret = select(r+1, &readfds, 0, 0, 0)) )) {
 	       if (FD_ISSET(r, &readfds)) {
                         char buf[1];
                         int n = read(r, buf, sizeof(buf));

@@ -186,6 +190,8 @@
                 readfds = fds;
         }
         gettimeofday(&now, 0);
+        fprintf(stderr, "USE_DOUBLE_SELECT=%d\n", USE_DOUBLE_SELECT);
+        fprintf(stderr, "misses=%u\n", misses);
         fprintf(stderr, "elapsed time for 100000 pingpongs is %g\n", now.tv_sec - then.tv_sec + (now.tv_usec -
 then.tv_usec) / 1000000.0);
         fprintf(stderr, "closing %d\n", r);

...I get constitently better results for "localhost 45644 45644 a" with USE_DOUBLE_SELECT=1 worth noting is that misses == 0 was always true. However if I have 2 programs, one doing "localhost 45642 45643" and one doing "localhost 45643 45642 a" then I get better results for USE_DOUBLE_SELECT=0[1] and misses is 80-90 thousand (Ie. it has to do 2 select calls 80-90 % of the time).

Please note that the original question was, select/poll does a small
schedule if you specify a timeout and that's bad. However in the two process case you _need_ the schedule, because there isn't any data there yet.
So again given the original assumtion that data is available on one
of the fd's then doing the double select is better, but if it isn't then you're wasting time no matter what you do.

As to why my test code got good results even though it uses 2
processes, I used PF_LOCAL/AF_LOCAL sockets not PF_INET/AF_INET and those are fast enough at transfering the data that you don't need the schedule (misses == 0, if you add similar code to above).

[1] This is on a real computer, on a 486 with 8Meg of RAM I still get better results with USE_DOUBLE_SELECT=1, and there are still 80% misses (no idea why).

# James Antill -- [email protected]
:0:
* ^From: .*james@and\.org
/dev/null