I/OAT DMA support and TCP acceleration

diff -rup a/net/ipv4/tcp.c b/net/ipv4/tcp.c
@@ -257,6 +257,7 @@
 #include <linux/fs.h>
 #include <linux/random.h>
 #include <linux/bootmem.h>
+#include <linux/dmaengine.h>
 
 #include <net/icmp.h>
 #include <net/tcp.h>

@@ -941,7 +942,7 @@ void tcp_cleanup_rbuf(struct sock *sk, i
 	struct tcp_sock *tp = tcp_sk(sk);
 	int time_to_ack = 0;
 
-#if TCP_DEBUG
+#if TCP_DEBUG && !defined(CONFIG_NET_DMA)
 	struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
 
 	BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));

@@ -1132,6 +1133,21 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 
 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
 
+#ifdef CONFIG_NET_DMA
+	if ((len > 1024) && !(flags & MSG_PEEK) && !sysctl_tcp_low_latency) {
+		preempt_disable();
+		tp->ucopy.dma_chan = __get_cpu_var(net_dma);
+		preempt_enable();
+	} else {
+		tp->ucopy.dma_chan = NULL;
+	}
+
+	if (tp->ucopy.dma_chan)
+		if (dma_lock_iovec_pages(msg->msg_iov, len,
+		                         &tp->ucopy.locked_list))
+			tp->ucopy.dma_chan = NULL;
+#endif
+
 	do {
 		struct sk_buff *skb;
 		u32 offset;

@@ -1161,7 +1177,12 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 				       "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
 				break;
 			}
-			offset = *seq - TCP_SKB_CB(skb)->seq;
+
+			if (!skb->copied_early)
+				offset = *seq - TCP_SKB_CB(skb)->seq;
+			else
+				offset = 0;
+
 			if (skb->h.th->syn)
 				offset--;
 			if (offset < skb->len)

@@ -1273,6 +1294,8 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 		} else
 			sk_wait_data(sk, &timeo);
 
+		tp->ucopy.wakeup = 0;
+
 		if (user_recv) {
 			int chunk;
 

@@ -1304,45 +1327,69 @@ do_prequeue:
 		}
 		continue;
 
-	found_ok_skb:
+found_ok_skb:
 		/* Ok so how much can we use? */
-		used = skb->len - offset;
-		if (len < used)
-			used = len;
-
-		/* Do we have urgent data here? */
-		if (tp->urg_data) {
-			u32 urg_offset = tp->urg_seq - *seq;
-			if (urg_offset < used) {
-				if (!urg_offset) {
-					if (!sock_flag(sk, SOCK_URGINLINE)) {
-						++*seq;
-						offset++;
-						used--;
-						if (!used)
-							goto skip_copy;
-					}
-				} else
-					used = urg_offset;
+		if (!skb->copied_early) {
+			used = skb->len - offset;
+			if (len < used)
+				used = len;
+
+			/* Do we have urgent data here? */
+			if (tp->urg_data) {
+				u32 urg_offset = tp->urg_seq - *seq;
+				if (urg_offset < used) {
+					if (!urg_offset) {
+						if (!sock_flag(sk, SOCK_URGINLINE)) {
+							++*seq;
+							offset++;
+							used--;
+							if (!used)
+								goto skip_copy;
+						}
+					} else
+						used = urg_offset;
+				}
 			}
-		}
 
-		if (!(flags & MSG_TRUNC)) {
-			err = skb_copy_datagram_iovec(skb, offset,
-						      msg->msg_iov, used);
-			if (err) {
-				/* Exception. Bailout! */
-				if (!copied)
-					copied = -EFAULT;
-				break;
+			if (!(flags & MSG_TRUNC)) {
+				if (tp->ucopy.dma_chan) {
+					tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
+						tp->ucopy.dma_chan, skb, offset,
+						msg->msg_iov, used,
+						tp->ucopy.locked_list);
+
+					if (tp->ucopy.dma_cookie < 0) {
+
+						printk(KERN_ALERT "dma_cookie < 0\n");
+
+						/* Exception. Bailout! */
+						if (!copied)
+							copied = -EFAULT;
+						break;
+					}
+					if ((offset + used) == skb->len)
+						skb->copied_early = 1;
+
+				} else {
+					err = skb_copy_datagram_iovec(skb, offset,
+							msg->msg_iov, used);
+					if (err) {
+						/* Exception. Bailout! */
+						if (!copied)
+							copied = -EFAULT;
+						break;
+					}
+				}
 			}
-		}
 
-		*seq += used;
-		copied += used;
-		len -= used;
+			*seq += used;
+			copied += used;
+			len -= used;
 
-		tcp_rcv_space_adjust(sk);
+			tcp_rcv_space_adjust(sk);
+		} else {
+			used = skb->len;
+		}
 
 skip_copy:
 		if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {

@@ -1354,15 +1401,27 @@ skip_copy:
 
 		if (skb->h.th->fin)
 			goto found_fin_ok;
-		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb);
+		if (!(flags & MSG_PEEK)) {
+			if (!skb->copied_early)
+				sk_eat_skb(sk, skb);
+			else {
+				__skb_unlink(skb, &sk->sk_receive_queue);
+				__skb_queue_tail(&tp->async_wait_queue, skb);
+			}
+		}
 		continue;
 
 	found_fin_ok:
 		/* Process the FIN. */
 		++*seq;
-		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb);
+		if (!(flags & MSG_PEEK)) {
+			if (!skb->copied_early)
+				sk_eat_skb(sk, skb);
+			else {
+				__skb_unlink(skb, &sk->sk_receive_queue);
+				__skb_queue_tail(&tp->async_wait_queue, skb);
+			}
+		}
 		break;
 	} while (len > 0);
 

@@ -1383,8 +1442,37 @@ skip_copy:
 
 		tp->ucopy.task = NULL;
 		tp->ucopy.len = 0;
+		tp->ucopy.bytes_early_copied = 0;
 	}
 
+
+#ifdef CONFIG_NET_DMA
+	if (tp->ucopy.dma_chan) {
+		struct sk_buff *skb;
+		dma_cookie_t done, used;
+
+		dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
+
+		while (dma_async_memcpy_complete(tp->ucopy.dma_chan,
+		                                 tp->ucopy.dma_cookie, &done,
+		                                 &used) == DMA_IN_PROGRESS) {
+			/* do partial cleanup of async_wait_queue */
+			while ((skb = skb_peek(&tp->async_wait_queue)) &&
+			       (dma_async_is_complete(TCP_SKB_CB(skb)->dma_cookie,
+			                       done, used) == DMA_SUCCESS)) {
+				__skb_dequeue(&tp->async_wait_queue);
+				kfree_skb(skb);
+			}
+		}
+
+		/* Safe to free early-copied skbs now */
+		__skb_queue_purge(&tp->async_wait_queue);
+		dma_unlock_iovec_pages(tp->ucopy.locked_list);
+		tp->ucopy.dma_chan = NULL;
+		tp->ucopy.locked_list = NULL;
+	}
+#endif
+
 	/* According to UNIX98, msg_name/msg_namelen are ignored
 	 * on connected socket. I was just happy when found this 8) --ANK
 	 */

106 lines not on this page · raw message

diff -rup a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
@@ -1148,6 +1148,11 @@ int tcp_v4_do_rcv(struct sock *sk, struc
 {
 	if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
 		TCP_CHECK_TIMER(sk);
+
+#ifdef CONFIG_NET_DMA
+		dma_async_try_early_copy(sk, skb);
+#endif
+
 		if (tcp_rcv_established(sk, skb, skb->h.th, skb->len))
 			goto reset;
 		TCP_CHECK_TIMER(sk);

@@ -1256,10 +1261,19 @@ process:
 	bh_lock_sock(sk);
 	ret = 0;
 	if (!sock_owned_by_user(sk)) {
-		if (!tcp_prequeue(sk, skb))
+#ifdef CONFIG_NET_DMA
+		struct tcp_sock *tp = tcp_sk(sk);
+		if (tp->ucopy.dma_chan)
 			ret = tcp_v4_do_rcv(sk, skb);
+		else
+#endif
+		{
+			if (!tcp_prequeue(sk, skb))
+			ret = tcp_v4_do_rcv(sk, skb);
+		}
 	} else
 		sk_add_backlog(sk, skb);
+
 	bh_unlock_sock(sk);
 
 	sock_put(sk);