Make TCP prequeue configurable
Subsystems (4)
RFC 1/1 Make TCP prequeue configurable
Hi all
I am sure some of you are going to tell me that prequeue is not all black :)
Thank you
[RFC] Make TCP prequeue configurable
The TCP prequeue thing is based on old facts, and has drawbacks.
1) It adds 48 bytes per 'struct tcp_sock'
2) It adds some ugly code in hot paths
3) It has a small hit ratio on typical servers using many sockets
4) It may have a high hit ratio on UP machines running one process, where the prequeue adds litle gain. (In fact, letting the user doing the copy after being woke up is better for cache reuse)
5) Doing a copy to user in softirq handler is not good, because of potential page faults :(
6) Maybe the NET_DMA thing is the only thing that might need prequeue.
This patch introduces a CONFIG_TCP_PREQUEUE, automatically selected if CONFIG_NET_DMA is on.
drivers/dma/Kconfig +1 -0
@@ -16,6 +16,7 @@ comment "DMA Clients" config NET_DMA bool "Network: TCP receive copy offload" depends on DMA_ENGINE && NET + select TCP_PREQUEUE default y ---help--- This enables the use of DMA engines in the network stack to
include/linux/tcp.h +2 -0
@@ -268,11 +268,13 @@ struct tcp_sock {/* Data for direct copy to user */ struct { +#ifdef CONFIG_TCP_PREQUEUE struct sk_buff_head prequeue;struct task_struct *task;struct iovec *iov;int memory;int len; +#endif #ifdef CONFIG_NET_DMA /* members for async copy */ struct dma_chan *dma_chan;
include/net/tcp.h +4 -0
@@ -835,10 +835,12 @@ static inline int tcp_checksum_complete(struct sk_buff *skb)static inline void tcp_prequeue_init (struct tcp_sock *tp) { +#ifdef CONFIG_TCP_PREQUEUE tp->ucopy.task =NULL ; tp->ucopy.len =0 ; tp->ucopy.memory =0 ;skb_queue_head_init (&tp->ucopy.prequeue); +#endif #ifdef CONFIG_NET_DMA tp->ucopy.dma_chan =NULL ; tp->ucopy.wakeup =0 ;
@@ -857,6 +859,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)*/ static inline int tcp_prequeue (struct sock *sk,struct sk_buff *skb) { +#ifdef CONFIG_TCP_PREQUEUE struct tcp_sock *tp =tcp_sk (sk);if (!sysctl_tcp_low_latency && tp->ucopy.task) {
@@ -882,6 +885,7 @@ static inline int tcp_prequeue(struct sock *sk, struct sk_buff *skb) }return 1 ; } +#endif return 0 ; }
net/ipv4/Kconfig +15 -0
@@ -616,5 +616,20 @@ config TCP_MD5SIG If unsure, say N. +config TCP_PREQUEUE + bool "Enable TCP prequeue" + default n + ---help--- + TCP PREQUEUE is an 'optimization' loosely based on the famous + "30 instruction TCP receive" Van Jacobson mail. + Van's trick is to deposit buffers into socket queue + on a device interrupt, to call tcp_recv function + on the receive process context and checksum and copy + the buffer to user space. smart... + + Some people believe this 'optimization' is not really needed + but for some benchmarks. Also, taking potential pagefaults in + softirq handler seems a high price to pay. + source "net/ipv4/ipvs/Kconfig"
net/ipv4/tcp.c +8 -1
@@ -994,6 +994,7 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied)tcp_send_ack (sk); } +#ifdef CONFIG_TCP_PREQUEUE static void tcp_prequeue_process (struct sock *sk) {struct sk_buff *skb;
@@ -1011,6 +1012,7 @@ static void tcp_prequeue_process(struct sock *sk)/* Clear memory counter. */ tp->ucopy.memory = 0; } +#endif static inline struct sk_buff *tcp_recv_skb (struct sock *sk,u32 seq,u32 *off) {
@@ -1251,6 +1253,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,tcp_cleanup_rbuf (sk, copied); +#ifdef CONFIG_TCP_PREQUEUE if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {/* Install new reader */ if (!user_recv && !(flags & (MSG_TRUNC |MSG_PEEK ))) {
@@ -1295,7 +1298,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, /* __ Set realtime policy in scheduler __ */ } - +#endif if (copied >= target) {/* Do not sleep, just process backlog. */ release_sock (sk);
@@ -1307,6 +1310,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, tp->ucopy.wakeup =0 ;#endif +#ifdef CONFIG_TCP_PREQUEUE if (user_recv) {int chunk;
@@ -1330,6 +1334,7 @@ do_prequeue: } } } +#endif if ((flags &MSG_PEEK ) && peek_seq != tp->copied_seq) {if (net_ratelimit ()) printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
@@ -1430,6 +1435,7 @@ skip_copy:break ; }while (len >0 ); +#ifdef CONFIG_TCP_PREQUEUE if (user_recv) {if (!skb_queue_empty (&tp->ucopy.prequeue)) {int chunk;
@@ -1448,6 +1454,7 @@ skip_copy: tp->ucopy.task =NULL ; tp->ucopy.len =0 ; } +#endif #ifdef CONFIG_NET_DMA if (tp->ucopy.dma_chan) {
net/ipv4/tcp_input.c +7 -2
@@ -3467,6 +3467,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) goto out_of_window;/* Ok. In sequence. In window. */ +#ifdef CONFIG_TCP_PREQUEUE if (tp->ucopy.task == current && tp->copied_seq == tp->rcv_nxt && tp->ucopy.len &&sock_owned_by_user (sk) && !tp->urg_data) {
@@ -3484,7 +3485,7 @@ static void tcp_data_queue(struct sock *sk, struct sk_buff *skb) }local_bh_disable (); } - +#endif if (eaten <=0 ) { queue_and_out: if (eaten < 0 &&
@@ -4078,6 +4079,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, struct tcphdr *th) } } +#ifdef CONFIG_TCP_PREQUEUE static int tcp_copy_to_iovec (struct sock *sk,struct sk_buff *skb,int hlen) {struct tcp_sock *tp =tcp_sk (sk);
@@ -4100,6 +4102,7 @@ static int tcp_copy_to_iovec(struct sock *sk, struct sk_buff *skb, int hlen)local_bh_disable (); return err; } +#endif static __sum16 __tcp_checksum_complete_user (struct sock *sk,struct sk_buff *skb) {
@@ -4279,8 +4282,9 @@ int tcp_rcv_established(struct sock *sk, struct sk_buff *skb, } }else {int eaten =0 ; -int copied_early =0 ; +#ifdef CONFIG_TCP_PREQUEUE +int copied_early =0 ;if (tp->copied_seq == tp->rcv_nxt && len - tcp_header_len <= tp->ucopy.len) {#ifdef CONFIG_NET_DMA
@@ -4315,6 +4319,7 @@ int tcp_rcv_established(struct sock *sk, struct sk_buff *skb,if (copied_early) tcp_cleanup_rbuf(sk, skb->len); } +#endif if (!eaten) {if (tcp_checksum_complete_user (sk, skb)) goto csum_error;
net/ipv4/tcp_ipv4.c +2 -0
@@ -1916,8 +1916,10 @@ int tcp_v4_destroy_sock(struct sock *sk)__skb_queue_purge (&sk->sk_async_wait_queue);#endif +#ifdef CONFIG_TCP_PREQUEUE /* Clean prequeue, it must be empty really */ __skb_queue_purge (&tp->ucopy.prequeue); +#endif /* Clean up a referenced TCP bind bucket. */ if (inet_csk (sk)->icsk_bind_hash)
net/ipv4/tcp_timer.c +4 -0
@@ -167,7 +167,9 @@ static int tcp_write_timeout(struct sock *sk)static void tcp_delack_timer (unsigned long data) {struct sock *sk = (struct sock *)data; +#ifdef CONFIG_TCP_PREQUEUE struct tcp_sock *tp =tcp_sk (sk); +#endif struct inet_connection_sock *icsk =inet_csk (sk);bh_lock_sock (sk);
@@ -190,6 +192,7 @@ static void tcp_delack_timer(unsigned long data) } icsk->icsk_ack.pending &= ~ICSK_ACK_TIMER ; +#ifdef CONFIG_TCP_PREQUEUE if (!skb_queue_empty (&tp->ucopy.prequeue)) {struct sk_buff *skb;
@@ -200,6 +203,7 @@ static void tcp_delack_timer(unsigned long data) tp->ucopy.memory = 0; } +#endif if (inet_csk_ack_scheduled (sk)) {if (!icsk->icsk_ack.pingpong) {
Quoted 18 lines · On Fri, 28 Sep 2007 00:08:33 +0200 Eric Dumazet <[email protected]> wrote:
Hi all
I am sure some of you are going to tell me that prequeue is not all black :)
Thank you
[RFC] Make TCP prequeue configurable
The TCP prequeue thing is based on old facts, and has drawbacks.
1) It adds 48 bytes per 'struct
tcp_sock'
2) It adds some ugly code in hot paths
3) It has a small hit ratio on typical servers using many sockets
4) It may have a high hit ratio on UP machines running one process, where the prequeue adds litle gain. (In fact, letting the user doing the copy after being woke up is better for cache reuse)
5) Doing a copy to user in softirq handler is not good, because of
Quoted 8 lines
potential page faults :(6) Maybe the
NET_DMAthing is the only thing that might need prequeue.This patch introduces a
CONFIG_TCP_PREQUEUE, automatically selected ifCONFIG_NET_DMAis on.Signed-off-by Eric Dumazet <[email protected]>
Rather than having a two more compile cases and test cases to deal with. If you can prove it is useless, make a case for killing it completely.
Signature · 1 line
Stephen Hemminger <[email protected]>
David Miller
maintainerFrom: Eric Dumazet <[email protected]>
Date: Fri, 28 Sep 2007 00:08:33 +0200
Quoted 9 lines
1) It adds 48 bytes per 'struct
tcp_sock'
2) It adds some ugly code in hot paths
3) It has a small hit ratio on typical servers using many sockets
4) It may have a high hit ratio on UP machines running one process, where the prequeue adds litle gain. (In fact, letting the user doing the copy after being woke up is better for cache reuse)
5) Doing a copy to user in softirq handler is not good, because ofpotential page faults :(6) Maybe the
NET_DMAthing is the only thing that might need prequeue.
If you want to make changes at least get your facts straight in your changelog message :-)
The prequeue doesn't do copies in softirqs, it acquires the user side socket lock and runs the packet input path directly from there, copying into userspace along the way.
You are making claims about performance based upon your understanding of the code and your understanding of typical workloads, rather than from actual measurements. In scientific communities this would make you a quack at best :-)
Quoted 24 lines · Stephen Hemminger
Quoted 21 lines · On Fri, 28 Sep 2007 00:08:33 +0200 Eric Dumazet <[email protected]> wrote:
Hi all
I am sure some of you are going to tell me that prequeue is not all black :)
Thank you
[RFC] Make TCP prequeue configurable
The TCP prequeue thing is based on old facts, and has drawbacks.
1) It adds 48 bytes per 'struct
tcp_sock'
2) It adds some ugly code in hot paths
3) It has a small hit ratio on typical servers using many sockets
4) It may have a high hit ratio on UP machines running one process, where the prequeue adds litle gain. (In fact, letting the user doing the copy after being woke up is better for cache reuse)
5) Doing a copy to user in softirq handler is not good, because ofpotential page faults :(6) Maybe the
NET_DMAthing is the only thing that might need prequeue.
Quoted 8 lines
Quoted 4 lines
This patch introduces a
CONFIG_TCP_PREQUEUE, automatically selected ifCONFIG_NET_DMAis on.Signed-off-by Eric Dumazet <[email protected]>Rather than having a two more compile cases and test cases to deal with. If you can prove it is useless, make a case for killing it completely.
I think it really does help in case (4) with old NICs that don't do rx checksumming. I'm not sure how many people really care about this anymore, but probably some...?
OTOH, it would be nice to get rid of sysctl_tcp_low_latency.
-John
David Miller
maintainerFrom: John Heffner <[email protected]>
Date: Thu, 27 Sep 2007 22:26:02 -0400
Quoted 5 lines
I think it really does help in case (4) with old NICs that don't do rx checksumming. I'm not sure how many people really care about this anymore, but probably some...?
OTOH, it would be nice to get rid of
sysctl_tcp_low_latency.
I know most high end apps use poll() so won't sleep in recvmsg() directly, but occasisionally they will, and even those that have a poll() triggered recvmsg() will run the backlog and do prequeue if packets arrive while they are processing the existing receive packets which is quite common.
So for any app that ends up doing a prequeue it's a win because there is the issue of scheduling and cpu usage charging.
If the ACK's are coming out of the stack at the rate that the application can pull data out of the receive queue, and no faster, this will pace the sender to send precisely how fast the receiver can get onto the cpu depending upon load.
Furthermore, prequeue puts the stack input processing work into user context, which means that the users will be charged more fairly for the work that is done for them.
When packets get fully processed in softirq context, that's bad because this is cpu usage which doesn't get charged to the user, and for TCP input processing this cpu usage is non-trivial and is multiplied by packet count.
Quoted 4 lines · David Miller
Furthermore, prequeue puts the stack input processing work into user context, which means that the users will be charged more fairly for the work that is done for them.
For more details on this people might want to read the old Lazy Receiver Processing papers: http://www.cs.rice.edu/CS/Systems/LRP/
-Andi