PROBLEM: small socket send/receive buffers on TCP stream result in data not being transferred
Subsystems (2)
I'm pretty sure this is a kernel problem
1. small socket send/receive buffers result in data not being transferred
2. I was playing with socket buffers using setsockopt() and SO_SNDBUF and SO_RCVBUF (see small attached programs), I found that given small enough buffers, data would stop being sent, although more was available. I suspect what is happening is that one of the buffers fills, stops accepting data, but doesn't reenable it when there's room again. When I use netstat I find the buffer on the send side full, and the buffer on the receive end empty.
This behaviour may depend on the speed of the system, on mine I can usually get it to happen with buffer sizes around 100, especially if the side sending data has a larger buffer.
I've tested this on a couple of machines running a 2.2 kernel, and one running a 2.4 kernel.
3. Keywords: Networking, TCP, kernel
4. Kernel version (output of /proc/version):
Linux version 2.4.0 (root@viola) (gcc version egcs-2.91.66 19990314/Linux (egcs-1.1.2 release)) #2 Thu Jan 25 19:56:28 NZDT 2001
6. Attached at the end are two C++ programs (though they could as easily have been C),
fakefeed - run first, acts as a server that just keeps sending data on a connection.
fakesim - run second, connects to fake feed, and outputs what's received. They both take as parameters a port number, and a buffer size. e.g. fakefeed 8001 500
fakesim 8001 100
7.1. output of ver_linux script:
-- Versions installed: (if some fields are empty or look -- unusual then possibly you have very old versions) Linux viola 2.4.0 #2 Thu Jan 25 19:56:28 NZDT 2001 i586 unknown
Kernel modules 2.4.0 Gnu C egcs-2.91.66 Gnu Make 3.77 Binutils 2.9.1.0.25 Linux C Library 2.1.2 Dynamic linker ldd: version 1.9.9 Procps 2.0.2 Mount 2.9v Net-tools 1.52 Kbd 0.99 Sh-utils 1.16 Modules Loaded ppp_deflate ipt_state ipt_MASQUERADE
iptable_filter ip_nat_ftp iptable_nat ip_conntrack ip_tables de4x5 v_midi opl3 mpu401 sb sb_lib uart401 sound bsd_comp ppp_async
7.2 output of /proc/cpuinfo
processor
: 0vendor_id
: GenuineIntel
cpu family : 5
model
: 4
model name : Pentium MMX
stepping
: 4
cpu MHz : 167.049fdiv_bug
: nohlt_bug
: nof00f_bug
: yescoma_bug
: no
fpu
: yesfpu_exception
: yes
cpuid level : 1
wp
: yes
flags
: fpu vme de pse tsc msr mce cx8 mmx
bogomips
: 333.41
7.3 output of /proc/modules
ppp_deflate 40912 0 (autoclean) ipt_state 800 2 (autoclean) ipt_MASQUERADE 1312 1 (autoclean) iptable_filter 1856 0 (autoclean) (unused) ip_nat_ftp 3440 0 (unused) iptable_nat 12608 1 [ipt_MASQUERADE ip_nat_ftp] ip_conntrack 12992 2 [ipt_state ipt_MASQUERADE ip_nat_ftp iptable_nat] ip_tables 10656 6 [ipt_state ipt_MASQUERADE iptable_filter iptable_nat] de4x5 42496 1 v_midi 5264 0 (unused) opl3 11792 0 (unused) mpu401 18960 0 (unused) sb 6656 0 (unused) sb_lib 34352 0 [sb] uart401 6416 0 [sb_lib] sound 57280 0 [v_midi opl3 mpu401 sb_lib uart401] bsd_comp 4160 0 ppp_async 6464 1
Thanks
Signature · 4 lines
bits and peace Nicholas Daley <mailto:[email protected]> insert :-)=s as appropriate
#include <stdlib.h>
#include <errno.h>
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>
#include <iostream.h>
int openList(int port) {
int fd=-1;
sockaddr_in sa;
fd=socket(PF_INET, SOCK_STREAM, 0);
if(fd<0) return -1;
sa.sin_family=AF_INET;
sa.sin_port=htons(port);
sa.sin_addr.s_addr=htonl(INADDR_ANY);
int s=1;
setsockopt(fd,SOL_SOCKET,SO_REUSEADDR,&s,sizeof(s));
if(bind(fd,(struct sockaddr*)&sa,sizeof(sa))<0) {
shutdown(fd,2);
fd=-1;
return fd;
}; if(listen(fd,1)<0) {
shutdown(fd,2);
fd=-1;
return fd; };
return fd;
};
int main(int argc, char** argv) {
int listener=openList(argc>1?atoi(argv[1]):8001); int stream=accept(listener,NULL,NULL); size_t s;
s=argc>2?atoi(argv[2]):3000; setsockopt(stream,SOL_SOCKET,SO_RCVBUF,&s,sizeof(s)); setsockopt(stream,SOL_SOCKET,SO_SNDBUF,&s,sizeof(s));
close(listener);
char buff[200];
long long sent=0;
for(int i=1;i;i++) {
sprintf(buff,"%d\n",i);
int writeres=write(stream,buff,strlen(buff));
if(writeres<0) { cerr<<"Write error "<<writeres<<','<<errno<<"\a\n";break;}
if(writeres==0) { cerr<<"Write EOF\a\n";break;}
if(writeres!=strlen(buff)) {cerr <<"Didn't finish writing\a\n";break;}
cerr<<buff<<" Total bytes sent="<<(sent+=writeres)<<endl;
}};
#include <stdlib.h>
#include <errno.h>
#include <netdb.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>
#include <iostream.h>
typedef double sk_time;
int openCon(const char* hostname, int port) {
int fd=-1;
sockaddr_in sa;
hostent *he;
fd=socket(PF_INET, SOCK_STREAM, 0); if(fd<0) return -1; sa.sin_family=AF_INET; sa.sin_port=htons(port); he=gethostbyname(hostname); if(he==NULL) return fd=-1; sa.sin_addr=*(struct in_addr*) he->h_addr; if(connect(fd,(struct sockaddr*) &sa, sizeof(sa))<0) return fd=-1;
return fd;
}
int main(int argc, char** argv) {
int tracestream=openCon("localhost",argc>1?atoi(argv[1]):8001); size_t s;
s=argc>2?atoi(argv[2]):3000; setsockopt(tracestream,SOL_SOCKET,SO_RCVBUF,&s,sizeof(s)); setsockopt(tracestream,SOL_SOCKET,SO_SNDBUF,&s,sizeof(s));
while(1) {
char c;
if(read(tracestream,&c,1)<=0) break;
cerr<<c; }
close(tracestream);
};
Hello!
Quoted 1 line
1. small socket send/receive buffers result in data not being transferred
I know why your test does not work in 2.4 (sort of bug, fix is appended).
But I have no idea, why it does not work with 2.2. Please, make tcpdump in this case.
Quoted 2 lines
and
SO_RCVBUF(see small attached programs), I found that given small enough buffers, data would stop being sent, although more was available.
No doubts. If you think that tcp must work like gun no matter what silly things application makes, you are mistaken. TCP works like gun when application does not try to interlope. 8)
You have made two capital mistakes:
1. RCVBUF/SNDBUF are not arbibrary numbers. Of course, kernel tries to adjust them to something reasonable, when user asks something wicked (like your case), but it cannot do this in all the cases.
2. Playing with buffer sizes after connection is established is very bad idea, tcp is defenceless in this case.
Quoted 1 line
This behaviour may depend on the speed of the system, on mine I can
Surely, it is not.
Quoted 1 line
usually get it to happen with buffer sizes around 100
Advice: do not play with these sockopts. Kernel does this better, than any application.
Application may set them, when it wants to set large values. Values, selected by kernel by default, are minimal ones in fact.
Alexey
include/net/tcp.h renamed from vger3-010130/linux/include/net/tcp.h +2 -2
@@ -1304,7 +1305,8 @@struct tcp_opt *tp = &sk->tp_pinfo.af_tcp;if (tp->ucopy.task) { -if ((tp->ucopy.memory += skb->truesize) <= (sk->rcvbuf<<1 )) { +if (tp->ucopy.memory < (sk->rcvbuf<<1 )) { + tp->ucopy.memory += skb->truesize;__skb_queue_tail (&tp->ucopy.prequeue, skb);if (skb_queue_len (&tp->ucopy.prequeue) ==1 ) {wake_up_interruptible (sk->sleep);
@@ -1313,7 +1315,6 @@ } }else {NET_INC_STATS_BH (TCPPrequeueDropped); - tp->ucopy.memory -= skb->truesize;__kfree_skb (skb); }return 1 ;
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
Please read the FAQ at http://www.tux.org/lkml/