Problem with e100 driver and latency on different packet sizes

I'm seeing an odd thing with the e100 driver. It seems to be this way with the 2.4 series and with 2.5.68, and I couldn't find anything with a search.

I've attached a small program to measure latency of round-trip time on UDP. If I send 85-byte packets between two of my machines, I get 170us round-trip latency. If I send 86-byte packets, I get 1329us latency. This seems quite odd. If I test on the eepro100 driver, I get expected linear increase in round-trip time as the packet size increases, and it never gets close to 1300us.

To run the program, do:

./ip_lat -s <port>

on one machine to run the server, and then do

./ip_lat <server IP> <port> 1000 85

to send 1000 85-byte packets from another machine. Change the 85 to 86 to see the latency go up.

-Corey

______________________________________________________________________

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/poll.h>
#include <sys/time.h>
#include <fcntl.h>
#include <netdb.h>
#include <malloc.h>
#include <errno.h>
#include <popt.h>

static int server;

struct poptOption poptOpts[]=

{
    {
	"server",
	's',
	POPT_ARG_NONE,
	NULL,
	's',
	"Enable the server",
	""
    },
    POPT_AUTOHELP
    {
	NULL,
	0,
	0,
	NULL,
	0		 
    }	
};

void
do_server(int port)

{
    int                fd;
    char               buffer[2048];
    struct sockaddr    addr;
    struct sockaddr_in ipaddr;
    socklen_t          addrlen;
    size_t             len;
    int                rv;

    fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
    if (fd == -1) {
	perror("Could not bind to port");
	exit(1);
    }

    ipaddr.sin_family = AF_INET;
    ipaddr.sin_port = htons(port);
    ipaddr.sin_addr.s_addr = INADDR_ANY;

    rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
    if (rv) {
	perror("Could not bind to port");
	exit(1);
    }

    for (;;) {
	addrlen = sizeof(addr);
	len = recvfrom(fd, buffer, sizeof(buffer), 0, &addr, &addrlen);
	if (len > 0) {
	    rv = sendto(fd, buffer, len, 0, &addr, addrlen);
	    if (rv == -1)
		perror("error in sendto");
	} else {
	    perror("Error in recvfrom");
	}
    }
}

static long
diff_timeval(struct timeval *left,
struct timeval *right)

{
    if (   (left->tv_sec < right->tv_sec)
	|| (   (left->tv_sec == right->tv_sec)
	    && (left->tv_usec < right->tv_usec)))
    {
	/* If left < right, just force to zero, don't allow negative
           numbers. */
	return 0;
    }

    return (((left->tv_sec - right->tv_sec) * 1000000)
	    + (left->tv_usec - right->tv_usec));
}

static long
average(long *data, int size, long *max, long *min)

{
    int  i;
    long rv = 0;

    *max = LONG_MIN;
    *min = LONG_MAX;
    for (i=0; i<size; i++) {
	if (data[i] < 0)
	    continue;
	if (data[i] > *max)
	    *max = data[i];
	if (data[i] < *min)
	    *min = data[i];
	rv += data[i];
    }

    return (rv / size);
}

void do_client(struct in_addr *dest_addr, int port, int count, int size)

{
    int                fd;
    struct sockaddr_in ipaddr;
    struct sockaddr    addr;
    socklen_t          addrlen;
    int                rv;
    int                i;
    size_t             len;
    char               buffer[2048];
    char               buffer2[2048];
    long               *times;
    long               avg, max, min;
    int                curr_port;

    times = malloc(sizeof(long) * count);
    if (!times) {
	fprintf(stderr, "Out of memory\n");
	exit(1);
    }

    fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
    if (fd == -1) {
	fprintf(stderr, "Out of memory\n");
	exit(1);
    }

    curr_port = 1000;
    do {
	curr_port++;
	ipaddr.sin_family = AF_INET;
	ipaddr.sin_port = htons(curr_port);
	ipaddr.sin_addr.s_addr = INADDR_ANY;

	rv = bind(fd, (struct sockaddr *) &ipaddr, sizeof(ipaddr));
    } while ((curr_port < 65536) && (rv == -1));

    ipaddr.sin_family = AF_INET;
    ipaddr.sin_port = htons(port);
    ipaddr.sin_addr = *dest_addr;

    for (i=0; i<size; i++) {
	buffer[i] = i;
    }

    for (i=0; i<count; i++) {
	struct timeval start_time;
	struct timeval end_time;
	struct timeval wait_time;
	fd_set read_set;

	times[i] = -1;

	FD_ZERO(&read_set);
	FD_SET(fd, &read_set);
	wait_time.tv_sec = 2;
	wait_time.tv_usec = 0;

	gettimeofday(&start_time, NULL);
	rv = sendto(fd, buffer, size, 0,
		    (struct sockaddr *) &ipaddr, sizeof(ipaddr));
	if (rv == -1) {
	    perror("Error in sendto");
	    continue;
	}
    retry:
	rv = select(fd+1, &read_set, NULL, NULL, &wait_time);
	gettimeofday(&end_time, NULL);
	if (rv == -1) {
	    if (errno == EINTR)
		goto retry;
	    perror("Error in select");
	    continue;
	}
	if (rv == 0) {
	    fprintf(stderr, "Timeout waiting for response %d\n", i);
	    continue;
	}

	addrlen = sizeof(addr);
	len = recvfrom(fd, buffer2, sizeof(buffer2), 0, &addr, &addrlen);
	if (rv == -1) {
	    perror("Error in recvfrom");
	    continue;
	}

	if (len != size) {
	    fprintf(stderr, "Invalid length in response to buffer %d,"
		    " waiting more\n", i);
	    goto retry;
	}
	if (memcmp(buffer, buffer2, size) != 0) {
	    fprintf(stderr, "Invalid data in response to buffer %d,"
		    " waiting more\n", i);
	    goto retry;
	}

	times[i] = diff_timeval(&end_time, &start_time);
    }

    avg = average(times, count, &max, &min);
    printf("Average: %ldus, Max: %ldus, Min: %ldus\n", avg, max, min);
}

int
main(int argc, const char *argv[])

{
    int            o;
    struct hostent *ent;
    struct in_addr addr;
    int            port;
    int            count;
    int            size;
    
    poptContext poptCtx = poptGetContext("ip_lat", argc, argv, poptOpts,0);

    while (( o = poptGetNextOpt(poptCtx)) >= 0)
    {   
	switch( o )
	{
	    case 's':
		server = 1;
		break;

	    default:
		poptPrintUsage(poptCtx, stderr, 0);
		exit(1);
	}
    }

    argv = poptGetArgs(poptCtx);

    if (!argv) {
	fprintf(stderr, "Not enough arguments\n");
	exit(1);
    }

    for (argc=0; argv[argc]!= NULL; argc++)
	;

    if ((server && (argc < 1)) || (!server && (argc < 4))) {
	fprintf(stderr, "Not enough arguments\n");
	exit(1);
    }

    if (server) {
	port = atoi(argv[0]);
	do_server(port);
    } else {
	ent = gethostbyname(argv[0]);
	if (!ent) {
	    fprintf(stderr, "gethostbyname failed: %s\n", strerror(h_errno));
	    exit(1);
	}
	memcpy(&addr, ent->h_addr_list[0], ent->h_length);
	port = atoi(argv[1]);

	count = atoi(argv[2]);
	size = atoi(argv[3]);
	do_client(&addr, port, count, size);
    }

    return 0;
}

sounds like the cpu cycle saver

check out networking/e100.txt

Quoted 23 lines

I'm seeing an odd thing with the e100 driver. It seems to be this way with the 2.4 series and with 2.5.68, and I couldn't find anything with a search.

I've attached a small program to measure latency of round-trip time on UDP. If I send 85-byte packets between two of my machines, I get 170us round-trip latency. If I send 86-byte packets, I get 1329us latency. This seems quite odd. If I test on the eepro100 driver, I get expected linear increase in round-trip time as the packet size increases, and it never gets close to 1300us.

To run the program, do:

./ip_lat -s <port>

on one machine to run the server, and then do

./ip_lat <server IP> <port> 1000 85

to send 1000 85-byte packets from another machine. Change the 85 to 86 to see the latency go up.

-Corey