Skip to contentSkip to conversation

I/OAT repost

PATCH In review Chris Leech 6 participants 21 messages May 24, 2006 – May 30, 2006

Subsystems (8)
lore
Share & tools
lore
#0 lore raw

cover I/OAT repost

This is a repost of the I/OAT patches, the only changes from last time are refreshing the patches and removing an unused macro that was causing the vger spam filters to drop patch 2/9.

This patch series is the a full release of the Intel(R) I/O Acceleration Technology (I/OAT) for Linux. It includes an in kernel API for offloading memory copies to hardware, a driver for the I/OAT DMA memcpy engine, and changes to the TCP stack to offload copies of received networking data to application space.

These changes apply to Linus' tree as of commit

	387e2b0439026aa738a9edca15a57e5c0bcb4dfc
	[BRIDGE]: need to ref count the LLC sap

They are available to pull from

	git://63.64.152.142/~cleech/linux-2.6 ioat-2.6.18

There are 9 patches in the series:
1) The memcpy offload APIs and class code
2) The Intel I/OAT DMA driver (ioatdma)
3) Core networking code to setup networking as a DMA memcpy client
4) Utility functions for sk_buff to iovec offloaded copy
5) Structure changes needed for TCP receive offload
6) Rename cleanup_rbuf to tcp_cleanup_rbuf
7) Make sk_eat_skb aware of early copied packets
8) Add a sysctl to tune the minimum offloaded I/O size for TCP
9) The main TCP receive offload changes

--
Chris Leech <[email protected]>
I/O Acceleration Technology Software Development
LAN Access Division / Digital Enterprise Group

#1 lore raw

PATCH 1/9 DMA memcpy subsystem

Provides an API for offloading memory copies to DMA devices

---

 drivers/Kconfig           |    2 
 drivers/Makefile          |    1 
 drivers/dma/Kconfig       |   13 +
 drivers/dma/Makefile      |    1 
 drivers/dma/dmaengine.c   |  408 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/dmaengine.h |  337 +++++++++++++++++++++++++++++++++++++
 6 files changed, 762 insertions(+), 0 deletions(-)
drivers/Kconfig +2 -0
index aeb5ab2..8b11ceb 100644
@@ -72,4 +72,6 @@ source "drivers/edac/Kconfig"
 
 source "drivers/rtc/Kconfig"
 
+source "drivers/dma/Kconfig"
+
 endmenu
drivers/Makefile +1 -0
index 447d8e6..3c51703 100644
@@ -74,3 +74,4 @@ obj-$(CONFIG_SGI_SN)		+= sn/
 obj-y				+= firmware/
 obj-$(CONFIG_CRYPTO)		+= crypto/
 obj-$(CONFIG_SUPERH)		+= sh/
+obj-$(CONFIG_DMA_ENGINE)	+= dma/
drivers/dma/Kconfig new file +13 -0
new file mode 100644
index 0000000..f9ac4bc
@@ -0,0 +1,13 @@
+#
+# DMA engine configuration
+#
+
+menu "DMA Engine support"
+
+config DMA_ENGINE
+	bool "Support for DMA engines"
+	---help---
+	  DMA engines offload copy operations from the CPU to dedicated
+	  hardware, allowing the copies to happen asynchronously.
+
+endmenu
drivers/dma/Makefile new file +1 -0
new file mode 100644
index 0000000..10b7391
@@ -0,0 +1 @@
+obj-y += dmaengine.o
drivers/dma/dmaengine.c new file +408 -0

408 lines not on this page · raw message

include/linux/dmaengine.h new file +337 -0

337 lines not on this page · raw message

#7 lore raw

PATCH 2/9 Driver for the Intel(R) I/OAT DMA engine

In reply to Chris Leech

Adds a new ioatdma driver

---

 drivers/dma/Kconfig             |    9 
 drivers/dma/Makefile            |    1 
 drivers/dma/ioatdma.c           |  839 +++++++++++++++++++++++++++++++++++++++
 drivers/dma/ioatdma.h           |  126 ++++++
 drivers/dma/ioatdma_hw.h        |   52 ++
 drivers/dma/ioatdma_io.h        |  118 +++++
 drivers/dma/ioatdma_registers.h |  126 ++++++
 7 files changed, 1271 insertions(+), 0 deletions(-)
drivers/dma/Kconfig +9 -0
index f9ac4bc..0f15e76 100644
@@ -10,4 +10,13 @@ config DMA_ENGINE
 	  DMA engines offload copy operations from the CPU to dedicated
 	  hardware, allowing the copies to happen asynchronously.
 
+comment "DMA Devices"
+
+config INTEL_IOATDMA
+	tristate "Intel I/OAT DMA support"
+	depends on DMA_ENGINE && PCI
+	default m
+	---help---
+	  Enable support for the Intel(R) I/OAT DMA engine.
+
 endmenu
drivers/dma/Makefile +1 -0
index 10b7391..c8a5f56 100644
@@ -1 +1,2 @@
 obj-y += dmaengine.o
+obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
drivers/dma/ioatdma.c new file +839 -0

839 lines not on this page · raw message

drivers/dma/ioatdma.h new file +126 -0

126 lines not on this page · raw message

drivers/dma/ioatdma_hw.h new file +52 -0

52 lines not on this page · raw message

drivers/dma/ioatdma_io.h new file +118 -0

118 lines not on this page · raw message

drivers/dma/ioatdma_registers.h new file +126 -0

126 lines not on this page · raw message

#11 lore raw

PATCH 3/9 Setup the networking subsystem as a DMA client

In reply to Chris Leech

Attempts to allocate per-CPU DMA channels

---

 drivers/dma/Kconfig       |   12 +++++
 include/linux/netdevice.h |    4 ++
 include/net/netdma.h      |   38 ++++++++++++++++
 net/core/dev.c            |  104 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 158 insertions(+), 0 deletions(-)
drivers/dma/Kconfig +12 -0
index 0f15e76..30d021d 100644
@@ -10,6 +10,18 @@ config DMA_ENGINE
 	  DMA engines offload copy operations from the CPU to dedicated
 	  hardware, allowing the copies to happen asynchronously.
 
+comment "DMA Clients"
+
+config NET_DMA
+	bool "Network: TCP receive copy offload"
+	depends on DMA_ENGINE && NET
+	default y
+	---help---
+	  This enables the use of DMA engines in the network stack to
+	  offload receive copy-to-user operations, freeing CPU cycles.
+	  Since this is the main user of the DMA engine, it should be enabled;
+	  say Y here.
+
 comment "DMA Devices"
 
 config INTEL_IOATDMA
include/linux/netdevice.h +4 -0
index f4169bb..b5760c6 100644
@@ -37,6 +37,7 @@
 #include <linux/config.h>
 #include <linux/device.h>
 #include <linux/percpu.h>
+#include <linux/dmaengine.h>
 
 struct divert_blk;
 struct vlan_group;
@@ -593,6 +594,9 @@ struct softnet_data
 	struct sk_buff		*completion_queue;
 
 	struct net_device	backlog_dev;	/* Sorry. 8) */
+#ifdef CONFIG_NET_DMA
+	struct dma_chan		*net_dma;
+#endif
 };
 
 DECLARE_PER_CPU(struct softnet_data,softnet_data);
include/net/netdma.h new file +38 -0
new file mode 100644
index 0000000..cbfe89d
@@ -0,0 +1,38 @@
+/*
+ * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef NETDMA_H
+#define NETDMA_H
+#include <linux/config.h>
+#ifdef CONFIG_NET_DMA
+#include <linux/dmaengine.h>
+
+static inline struct dma_chan *get_softnet_dma(void)
+{
+	struct dma_chan *chan;
+	rcu_read_lock();
+	chan = rcu_dereference(__get_cpu_var(softnet_data.net_dma));
+	if (chan)
+		dma_chan_get(chan);
+	rcu_read_unlock();
+	return chan;
+}
+#endif /* CONFIG_NET_DMA */
+#endif /* NETDMA_H */
net/core/dev.c +104 -0

134 lines not on this page · raw message

#13 lore raw

PATCH 4/9 Utility functions for offloading sk_buff to iovec copies

In reply to Chris Leech

Provides for pinning user space pages in memory, copying to iovecs, and copying from sk_buffs including fragmented and chained sk_buffs.

---

 drivers/dma/Makefile      |    3 
 drivers/dma/iovlock.c     |  301 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/dmaengine.h |   22 +++
 include/net/netdma.h      |    6 +
 net/core/Makefile         |    1 
 net/core/user_dma.c       |  127 +++++++++++++++++++
 6 files changed, 459 insertions(+), 1 deletions(-)
drivers/dma/Makefile +2 -1
index c8a5f56..bdcfdbd 100644
@@ -1,2 +1,3 @@
-obj-y += dmaengine.o
+obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
+obj-$(CONFIG_NET_DMA) += iovlock.o
 obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
drivers/dma/iovlock.c new file +301 -0

301 lines not on this page · raw message

include/linux/dmaengine.h +22 -0
index 3078154..78b236c 100644
@@ -333,5 +333,27 @@ static inline enum dma_status dma_async_
 int dma_async_device_register(struct dma_device *device);
 void dma_async_device_unregister(struct dma_device *device);
 
+/* --- Helper iov-locking functions --- */
+
+struct dma_page_list {
+	char *base_address;
+	int nr_pages;
+	struct page **pages;
+};
+
+struct dma_pinned_list {
+	int nr_iovecs;
+	struct dma_page_list page_list[0];
+};
+
+struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
+void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
+
+dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
+	struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
+dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
+	struct dma_pinned_list *pinned_list, struct page *page,
+	unsigned int offset, size_t len);
+
 #endif /* CONFIG_DMA_ENGINE */
 #endif /* DMAENGINE_H */
include/net/netdma.h +6 -0
index cbfe89d..19760eb 100644
@@ -23,6 +23,7 @@
 #include <linux/config.h>
 #ifdef CONFIG_NET_DMA
 #include <linux/dmaengine.h>
+#include <linux/skbuff.h>
 
 static inline struct dma_chan *get_softnet_dma(void)
 {
@@ -34,5 +35,10 @@ static inline struct dma_chan *get_softn
 	rcu_read_unlock();
 	return chan;
 }
+
+int dma_skb_copy_datagram_iovec(struct dma_chan* chan,
+		const struct sk_buff *skb, int offset, struct iovec *to,
+		size_t len, struct dma_pinned_list *pinned_list);
+
 #endif /* CONFIG_NET_DMA */
 #endif /* NETDMA_H */
net/core/Makefile +1 -0
index 79fe12c..e9bd246 100644
@@ -16,3 +16,4 @@ obj-$(CONFIG_NET_DIVERT) += dv.o
 obj-$(CONFIG_NET_PKTGEN) += pktgen.o
 obj-$(CONFIG_WIRELESS_EXT) += wireless.o
 obj-$(CONFIG_NETPOLL) += netpoll.o
+obj-$(CONFIG_NET_DMA) += user_dma.o
net/core/user_dma.c new file +127 -0

127 lines not on this page · raw message

#15 lore raw

PATCH 6/9 Rename cleanup_rbuf to tcp_cleanup_rbuf and make non-static

In reply to Chris Leech

Needed to be able to call tcp_cleanup_rbuf in tcp_input.c for I/OAT

---

 include/net/tcp.h |    2 ++
 net/ipv4/tcp.c    |   10 +++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)
include/net/tcp.h +2 -0
index d0c2c2f..578cccf 100644
@@ -294,6 +294,8 @@ extern int			tcp_rcv_established(struct 
 
 extern void			tcp_rcv_space_adjust(struct sock *sk);
 
+extern void			tcp_cleanup_rbuf(struct sock *sk, int copied);
+
 extern int			tcp_twsk_unique(struct sock *sk,
 						struct sock *sktw, void *twp);
 
net/ipv4/tcp.c +5 -5
index e2b7b80..1c0cfd7 100644
@@ -937,7 +937,7 @@ static int tcp_recv_urg(struct sock *sk,
  * calculation of whether or not we must ACK for the sake of
  * a window update.
  */
-static void cleanup_rbuf(struct sock *sk, int copied)
+void tcp_cleanup_rbuf(struct sock *sk, int copied)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	int time_to_ack = 0;
@@ -1086,7 +1086,7 @@ int tcp_read_sock(struct sock *sk, read_
 
 	/* Clean up data we have read: This will do ACK frames. */
 	if (copied)
-		cleanup_rbuf(sk, copied);
+		tcp_cleanup_rbuf(sk, copied);
 	return copied;
 }
 
@@ -1220,7 +1220,7 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 			}
 		}
 
-		cleanup_rbuf(sk, copied);
+		tcp_cleanup_rbuf(sk, copied);
 
 		if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
 			/* Install new reader */
@@ -1391,7 +1391,7 @@ skip_copy:
 	 */
 
 	/* Clean up data we have read: This will do ACK frames. */
-	cleanup_rbuf(sk, copied);
+	tcp_cleanup_rbuf(sk, copied);
 
 	TCP_CHECK_TIMER(sk);
 	release_sock(sk);
@@ -1858,7 +1858,7 @@ static int do_tcp_setsockopt(struct sock
 			    (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
 			    inet_csk_ack_scheduled(sk)) {
 				icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
-				cleanup_rbuf(sk, 1);
+				tcp_cleanup_rbuf(sk, 1);
 				if (!(val & 1))
 					icsk->icsk_ack.pingpong = 1;
 			}
#16 lore raw

PATCH 7/9 make sk_eat_skb I/OAT aware

In reply to Chris Leech

Add an extra argument to sk_eat_skb, and make it move early copied packets to the async_wait_queue instead of freeing them.

---

 include/net/sock.h |   13 ++++++++++++-
 net/dccp/proto.c   |    4 ++--
 net/ipv4/tcp.c     |    8 ++++----
 net/llc/af_llc.c   |    2 +-
 4 files changed, 19 insertions(+), 8 deletions(-)
include/net/sock.h +12 -1
index 90c65cb..75b0e97 100644
@@ -1273,11 +1273,22 @@ sock_recv_timestamp(struct msghdr *msg, 
  * This routine must be called with interrupts disabled or with the socket
  * locked so that the sk_buff queue operation is ok.
 */
-static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
+#ifdef CONFIG_NET_DMA
+static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, int copied_early)
+{
+	__skb_unlink(skb, &sk->sk_receive_queue);
+	if (!copied_early)
+		__kfree_skb(skb);
+	else
+		__skb_queue_tail(&sk->sk_async_wait_queue, skb);
+}
+#else
+static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, int copied_early)
 {
 	__skb_unlink(skb, &sk->sk_receive_queue);
 	__kfree_skb(skb);
 }
+#endif
 
 extern void sock_enable_timestamp(struct sock *sk);
 extern int sock_get_timestamp(struct sock *, struct timeval __user *);
net/dccp/proto.c +2 -2
index 2e0ee83..5317fd3 100644
@@ -719,7 +719,7 @@ int dccp_recvmsg(struct kiocb *iocb, str
 		}
 		dccp_pr_debug("packet_type=%s\n",
 			      dccp_packet_name(dh->dccph_type));
-		sk_eat_skb(sk, skb);
+		sk_eat_skb(sk, skb, 0);
 verify_sock_status:
 		if (sock_flag(sk, SOCK_DONE)) {
 			len = 0;
@@ -773,7 +773,7 @@ verify_sock_status:
 		}
 	found_fin_ok:
 		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb);
+			sk_eat_skb(sk, skb, 0);
 		break;
 	} while (1);
 out:
net/ipv4/tcp.c +4 -4
index 1c0cfd7..4e067d2 100644
@@ -1072,11 +1072,11 @@ int tcp_read_sock(struct sock *sk, read_
 				break;
 		}
 		if (skb->h.th->fin) {
-			sk_eat_skb(sk, skb);
+			sk_eat_skb(sk, skb, 0);
 			++seq;
 			break;
 		}
-		sk_eat_skb(sk, skb);
+		sk_eat_skb(sk, skb, 0);
 		if (!desc->count)
 			break;
 	}
@@ -1356,14 +1356,14 @@ skip_copy:
 		if (skb->h.th->fin)
 			goto found_fin_ok;
 		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb);
+			sk_eat_skb(sk, skb, 0);
 		continue;
 
 	found_fin_ok:
 		/* Process the FIN. */
 		++*seq;
 		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb);
+			sk_eat_skb(sk, skb, 0);
 		break;
 	} while (len > 0);
 
net/llc/af_llc.c +1 -1
index 5a04db7..7465170 100644
@@ -789,7 +789,7 @@ static int llc_ui_recvmsg(struct kiocb *
 			continue;
 
 		if (!(flags & MSG_PEEK)) {
-			sk_eat_skb(sk, skb);
+			sk_eat_skb(sk, skb, 0);
 			*seq = 0;
 		}
 	} while (len > 0);
#17 lore raw

PATCH 8/9 Add a sysctl for tuning the I/OAT offloaded I/O threshold

In reply to Chris Leech

Any socket recv of less than this ammount will not be offloaded

---

 include/linux/sysctl.h     |    1 +
 include/net/tcp.h          |    1 +
 net/core/user_dma.c        |    4 ++++
 net/ipv4/sysctl_net_ipv4.c |   10 ++++++++++
 4 files changed, 16 insertions(+), 0 deletions(-)
include/linux/sysctl.h +1 -0
index 76eaeff..cd9e7c0 100644
@@ -403,6 +403,7 @@ enum
  	NET_TCP_MTU_PROBING=113,
 	NET_TCP_BASE_MSS=114,
 	NET_IPV4_TCP_WORKAROUND_SIGNED_WINDOWS=115,
+	NET_TCP_DMA_COPYBREAK=116,
 };
 
 enum {
include/net/tcp.h +1 -0
index 578cccf..f1f4727 100644
@@ -219,6 +219,7 @@ extern int sysctl_tcp_adv_win_scale;
 extern int sysctl_tcp_tw_reuse;
 extern int sysctl_tcp_frto;
 extern int sysctl_tcp_low_latency;
+extern int sysctl_tcp_dma_copybreak;
 extern int sysctl_tcp_nometrics_save;
 extern int sysctl_tcp_moderate_rcvbuf;
 extern int sysctl_tcp_tso_win_divisor;
net/core/user_dma.c +4 -0
index 9eee91b..b7c98db 100644
@@ -30,6 +30,10 @@
 #include <linux/rtnetlink.h> /* for BUG_TRAP */
 #include <net/tcp.h>
 
+#define NET_DMA_DEFAULT_COPYBREAK 4096
+
+int sysctl_tcp_dma_copybreak = NET_DMA_DEFAULT_COPYBREAK;
+
 /**
  *	dma_skb_copy_datagram_iovec - Copy a datagram to an iovec.
  *	@skb - buffer to copy
net/ipv4/sysctl_net_ipv4.c +10 -0
index 6b6c3ad..6a6aa53 100644
@@ -688,6 +688,16 @@ ctl_table ipv4_table[] = {
 		.mode		= 0644,
 		.proc_handler	= &proc_dointvec
 	},
+#ifdef CONFIG_NET_DMA
+	{
+		.ctl_name	= NET_TCP_DMA_COPYBREAK,
+		.procname	= "tcp_dma_copybreak",
+		.data		= &sysctl_tcp_dma_copybreak,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec
+	},
+#endif
 	{ .ctl_name = 0 }
 };
 
#19 lore raw

PATCH 9/9 TCP recv offload to I/OAT

In reply to Chris Leech

Locks down user pages and sets up for DMA in tcp_recvmsg, then calls dma_async_try_early_copy in tcp_v4_do_rcv

---

 net/ipv4/tcp.c       |  103 ++++++++++++++++++++++++++++++++++++++++++++------
 net/ipv4/tcp_input.c |   74 +++++++++++++++++++++++++++++++++---
 net/ipv4/tcp_ipv4.c  |   18 ++++++++-
 net/ipv6/tcp_ipv6.c  |   12 +++++-
 4 files changed, 185 insertions(+), 22 deletions(-)
net/ipv4/tcp.c +91 -12
index 4e067d2..ff6ccda 100644
@@ -263,7 +263,7 @@
 #include <net/tcp.h>
 #include <net/xfrm.h>
 #include <net/ip.h>
-
+#include <net/netdma.h>
 
 #include <asm/uaccess.h>
 #include <asm/ioctls.h>
@@ -1110,6 +1110,7 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 	int target;		/* Read at least this many bytes */
 	long timeo;
 	struct task_struct *user_recv = NULL;
+	int copied_early = 0;
 
 	lock_sock(sk);
 
@@ -1133,6 +1134,17 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 
 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
 
+#ifdef CONFIG_NET_DMA
+	tp->ucopy.dma_chan = NULL;
+	preempt_disable();
+	if ((len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
+	    !sysctl_tcp_low_latency && __get_cpu_var(softnet_data.net_dma)) {
+		preempt_enable_no_resched();
+		tp->ucopy.pinned_list = dma_pin_iovec_pages(msg->msg_iov, len);
+	} else
+		preempt_enable_no_resched();
+#endif
+
 	do {
 		struct sk_buff *skb;
 		u32 offset;
@@ -1274,6 +1286,10 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 		} else
 			sk_wait_data(sk, &timeo);
 
+#ifdef CONFIG_NET_DMA
+		tp->ucopy.wakeup = 0;
+#endif
+
 		if (user_recv) {
 			int chunk;
 
@@ -1329,13 +1345,39 @@ do_prequeue:
 		}
 
 		if (!(flags & MSG_TRUNC)) {
-			err = skb_copy_datagram_iovec(skb, offset,
-						      msg->msg_iov, used);
-			if (err) {
-				/* Exception. Bailout! */
-				if (!copied)
-					copied = -EFAULT;
-				break;
+#ifdef CONFIG_NET_DMA
+			if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
+				tp->ucopy.dma_chan = get_softnet_dma();
+
+			if (tp->ucopy.dma_chan) {
+				tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
+					tp->ucopy.dma_chan, skb, offset,
+					msg->msg_iov, used,
+					tp->ucopy.pinned_list);
+
+				if (tp->ucopy.dma_cookie < 0) {
+
+					printk(KERN_ALERT "dma_cookie < 0\n");
+
+					/* Exception. Bailout! */
+					if (!copied)
+						copied = -EFAULT;
+					break;
+				}
+				if ((offset + used) == skb->len)
+					copied_early = 1;
+
+			} else
+#endif
+			{
+				err = skb_copy_datagram_iovec(skb, offset,
+						msg->msg_iov, used);
+				if (err) {
+					/* Exception. Bailout! */
+					if (!copied)
+						copied = -EFAULT;
+					break;
+				}
 			}
 		}
 
@@ -1355,15 +1397,19 @@ skip_copy:
 
 		if (skb->h.th->fin)
 			goto found_fin_ok;
-		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb, 0);
+		if (!(flags & MSG_PEEK)) {
+			sk_eat_skb(sk, skb, copied_early);
+			copied_early = 0;
+		}
 		continue;
 
 	found_fin_ok:
 		/* Process the FIN. */
 		++*seq;
-		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb, 0);
+		if (!(flags & MSG_PEEK)) {
+			sk_eat_skb(sk, skb, copied_early);
+			copied_early = 0;
+		}
 		break;
 	} while (len > 0);
 
@@ -1386,6 +1432,36 @@ skip_copy:
 		tp->ucopy.len = 0;
 	}
 
+#ifdef CONFIG_NET_DMA
+	if (tp->ucopy.dma_chan) {
+		struct sk_buff *skb;
+		dma_cookie_t done, used;
+
+		dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
+
+		while (dma_async_memcpy_complete(tp->ucopy.dma_chan,
+		                                 tp->ucopy.dma_cookie, &done,
+		                                 &used) == DMA_IN_PROGRESS) {
+			/* do partial cleanup of sk_async_wait_queue */
+			while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
+			       (dma_async_is_complete(skb->dma_cookie, done,
+			                              used) == DMA_SUCCESS)) {
+				__skb_dequeue(&sk->sk_async_wait_queue);
+				kfree_skb(skb);
+			}
+		}
+
+		/* Safe to free early-copied skbs now */
+		__skb_queue_purge(&sk->sk_async_wait_queue);
+		dma_chan_put(tp->ucopy.dma_chan);
+		tp->ucopy.dma_chan = NULL;
+	}
+	if (tp->ucopy.pinned_list) {
+		dma_unpin_iovec_pages(tp->ucopy.pinned_list);
+		tp->ucopy.pinned_list = NULL;
+	}
+#endif
+
 	/* According to UNIX98, msg_name/msg_namelen are ignored
 	 * on connected socket. I was just happy when found this 8) --ANK
 	 */
@@ -1658,6 +1734,9 @@ int tcp_disconnect(struct sock *sk, int 
 	__skb_queue_purge(&sk->sk_receive_queue);
 	sk_stream_writequeue_purge(sk);
 	__skb_queue_purge(&tp->out_of_order_queue);
+#ifdef CONFIG_NET_DMA
+	__skb_queue_purge(&sk->sk_async_wait_queue);
+#endif
 
 	inet->dport = 0;
 
net/ipv4/tcp_input.c +67 -7

107 lines not on this page · raw message

net/ipv4/tcp_ipv4.c +17 -1
index 672950e..25ecc6e 100644
@@ -71,6 +71,7 @@
 #include <net/inet_common.h>
 #include <net/timewait_sock.h>
 #include <net/xfrm.h>
+#include <net/netdma.h>
 
 #include <linux/inet.h>
 #include <linux/ipv6.h>
@@ -1091,8 +1092,18 @@ process:
 	bh_lock_sock(sk);
 	ret = 0;
 	if (!sock_owned_by_user(sk)) {
-		if (!tcp_prequeue(sk, skb))
+#ifdef CONFIG_NET_DMA
+		struct tcp_sock *tp = tcp_sk(sk);
+		if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
+			tp->ucopy.dma_chan = get_softnet_dma();
+		if (tp->ucopy.dma_chan)
 			ret = tcp_v4_do_rcv(sk, skb);
+		else
+#endif
+		{
+			if (!tcp_prequeue(sk, skb))
+			ret = tcp_v4_do_rcv(sk, skb);
+		}
 	} else
 		sk_add_backlog(sk, skb);
 	bh_unlock_sock(sk);
@@ -1296,6 +1307,11 @@ int tcp_v4_destroy_sock(struct sock *sk)
 	/* Cleans up our, hopefully empty, out_of_order_queue. */
   	__skb_queue_purge(&tp->out_of_order_queue);
 
+#ifdef CONFIG_NET_DMA
+	/* Cleans up our sk_async_wait_queue */
+  	__skb_queue_purge(&sk->sk_async_wait_queue);
+#endif
+
 	/* Clean prequeue, it must be empty really */
 	__skb_queue_purge(&tp->ucopy.prequeue);
 
net/ipv6/tcp_ipv6.c +10 -2
index 301eee7..a50eb30 100644
@@ -1218,8 +1218,16 @@ process:
 	bh_lock_sock(sk);
 	ret = 0;
 	if (!sock_owned_by_user(sk)) {
-		if (!tcp_prequeue(sk, skb))
-			ret = tcp_v6_do_rcv(sk, skb);
+#ifdef CONFIG_NET_DMA
+                struct tcp_sock *tp = tcp_sk(sk);
+                if (tp->ucopy.dma_chan)
+                        ret = tcp_v6_do_rcv(sk, skb);
+                else
+#endif
+		{
+			if (!tcp_prequeue(sk, skb))
+				ret = tcp_v6_do_rcv(sk, skb);
+		}
 	} else
 		sk_add_backlog(sk, skb);
 	bh_unlock_sock(sk);
#14 lore raw

PATCH 5/9 Structure changes for TCP recv offload to I/OAT

In reply to Chris Leech

Adds an async_wait_queue and some additional fields to tcp_sock, and a dma_cookie_t to sk_buff.

---

 include/linux/skbuff.h |    4 ++++
 include/linux/tcp.h    |    8 ++++++++
 include/net/sock.h     |    2 ++
 include/net/tcp.h      |    7 +++++++
 net/core/sock.c        |    6 ++++++
 5 files changed, 27 insertions(+), 0 deletions(-)
include/linux/skbuff.h +4 -0
index f8f2347..23bad3b 100644
@@ -29,6 +29,7 @@
 #include <linux/net.h>
 #include <linux/textsearch.h>
 #include <net/checksum.h>
+#include <linux/dmaengine.h>
 
 #define HAVE_ALLOC_SKB		/* For the drivers to know */
 #define HAVE_ALIGNABLE_SKB	/* Ditto 8)		   */
@@ -285,6 +286,9 @@ struct sk_buff {
 	__u16			tc_verd;	/* traffic control verdict */
 #endif
 #endif
+#ifdef CONFIG_NET_DMA
+	dma_cookie_t		dma_cookie;
+#endif
 
 
 	/* These elements must be at the end, see alloc_skb() for details.  */
include/linux/tcp.h +8 -0
index 542d395..c90daa5 100644
@@ -18,6 +18,7 @@
 #define _LINUX_TCP_H
 
 #include <linux/types.h>
+#include <linux/dmaengine.h>
 #include <asm/byteorder.h>
 
 struct tcphdr {
@@ -233,6 +234,13 @@ struct tcp_sock {
 		struct iovec		*iov;
 		int			memory;
 		int			len;
+#ifdef CONFIG_NET_DMA
+		/* members for async copy */
+		struct dma_chan		*dma_chan;
+		int			wakeup;
+		struct dma_pinned_list	*pinned_list;
+		dma_cookie_t		dma_cookie;
+#endif
 	} ucopy;
 
 	__u32	snd_wl1;	/* Sequence for window update		*/
include/net/sock.h +2 -0
index c9fad6f..90c65cb 100644
@@ -132,6 +132,7 @@ struct sock_common {
   *	@sk_receive_queue: incoming packets
   *	@sk_wmem_alloc: transmit queue bytes committed
   *	@sk_write_queue: Packet sending queue
+  *	@sk_async_wait_queue: DMA copied packets
   *	@sk_omem_alloc: "o" is "option" or "other"
   *	@sk_wmem_queued: persistent queue size
   *	@sk_forward_alloc: space allocated forward
@@ -205,6 +206,7 @@ struct sock {
 	atomic_t		sk_omem_alloc;
 	struct sk_buff_head	sk_receive_queue;
 	struct sk_buff_head	sk_write_queue;
+	struct sk_buff_head	sk_async_wait_queue;
 	int			sk_wmem_queued;
 	int			sk_forward_alloc;
 	gfp_t			sk_allocation;
include/net/tcp.h +7 -0
index 3c989db..d0c2c2f 100644
@@ -28,6 +28,7 @@
 #include <linux/cache.h>
 #include <linux/percpu.h>
 #include <linux/skbuff.h>
+#include <linux/dmaengine.h>
 
 #include <net/inet_connection_sock.h>
 #include <net/inet_timewait_sock.h>
@@ -817,6 +818,12 @@ static inline void tcp_prequeue_init(str
 	tp->ucopy.len = 0;
 	tp->ucopy.memory = 0;
 	skb_queue_head_init(&tp->ucopy.prequeue);
+#ifdef CONFIG_NET_DMA
+	tp->ucopy.dma_chan = NULL;
+	tp->ucopy.wakeup = 0;
+	tp->ucopy.pinned_list = NULL;
+	tp->ucopy.dma_cookie = 0;
+#endif
 }
 
 /* Packet is added to VJ-style prequeue for processing in process
net/core/sock.c +6 -0
index ed2afdb..5d820c3 100644
@@ -832,6 +832,9 @@ struct sock *sk_clone(const struct sock 
 		atomic_set(&newsk->sk_omem_alloc, 0);
 		skb_queue_head_init(&newsk->sk_receive_queue);
 		skb_queue_head_init(&newsk->sk_write_queue);
+#ifdef CONFIG_NET_DMA
+		skb_queue_head_init(&newsk->sk_async_wait_queue);
+#endif
 
 		rwlock_init(&newsk->sk_dst_lock);
 		rwlock_init(&newsk->sk_callback_lock);
@@ -1383,6 +1386,9 @@ void sock_init_data(struct socket *sock,
 	skb_queue_head_init(&sk->sk_receive_queue);
 	skb_queue_head_init(&sk->sk_write_queue);
 	skb_queue_head_init(&sk->sk_error_queue);
+#ifdef CONFIG_NET_DMA
+	skb_queue_head_init(&sk->sk_async_wait_queue);
+#endif
 
 	sk->sk_send_head	=	NULL;
 
maintainer
#20 lore raw
In reply to Chris Leech

From: Chris Leech <[email protected]>
Subject: [PATCH 0/9] I/OAT repost
Date: Tue, 23 May 2006 17:16:53 -0700

Quoted 9 lines

This is a repost of the I/OAT patches, the only changes from last time are refreshing the patches and removing an unused macro that was causing the vger spam filters to drop patch 2/9.

This patch series is the a full release of the Intel(R) I/O Acceleration Technology (I/OAT) for Linux. It includes an in kernel API for offloading memory copies to hardware, a driver for the I/OAT DMA memcpy engine, and changes to the TCP stack to offload copies of received networking data to application space.

I'm going to apply this into a net-2.6.18 GIT tree, do some build and sanity checking, then ask Andrew to pull it into -mm for testing.

Thanks guys.

#2 lore raw

Re: [PATCH 1/9] [I/OAT] DMA memcpy subsystem

In reply to Chris Leech

Review

Quoted 2 lines · Chris Leech
(quoted diff) +1 -0
+	for_each_cpu(i)

That's about to be deleted. Please use for_each_possible_cpu().

That's if for_each_possible_cpu() is appropriate. Perhaps it should be using for_each_present_cpu(), or for_each_online_cpu(). That's why for_each_cpu() is going away - to make people think about such things..

#4 lore raw

Re: [PATCH 1/9] [I/OAT] DMA memcpy subsystem

In reply to Chris Leech
Quoted 22 lines · Chris Leech

+/**
+ * dma_client_chan_free - release a DMA channel
+ * @chan: &dma_chan
+ */
+void dma_chan_cleanup(struct kref *kref)
+{
+ struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
+ chan->device->device_free_chan_resources(chan);
+ chan->client = NULL;
+ kref_put(&chan->device->refcount, dma_async_device_cleanup);
+}
+
+static void dma_chan_free_rcu(struct rcu_head *rcu) +{
+ struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
+ int bias = 0x7FFFFFFF;
+ int i;
+ for_each_cpu(i)
+ bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount);
+ atomic_sub(bias, &chan->refcount.refcount);
+ kref_put(&chan->refcount, dma_chan_cleanup);

Quoted 8 lines

+}
+
+static void dma_client_chan_free(struct dma_chan *chan) +{
+ atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
+ chan->slow_ref = 1;
+ call_rcu(&chan->rcu, dma_chan_free_rcu);
+}

A comment describing this `bias' magic would be nice.

#8 lore raw

Re: [PATCH 2/9] [I/OAT] Driver for the Intel(R) I/OAT DMA engine

In reply to Chris Leech
Quoted 12 lines · Chris Leech

+static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
+{
+ unsigned long phys_complete;
+ struct ioat_desc_sw *desc, *_desc;
+ dma_cookie_t cookie = 0;
+
+ prefetch(chan->completion_virt);
+
+ if (!spin_trylock(&chan->cleanup_lock))
+ return;
+

spin_trylock() is a red flag. It often means that someone screwed their locking up.

It at least needs a comment explaining its presence.

#18 lore raw

Re: [PATCH 8/9] [I/OAT] Add a sysctl for tuning the I/OAT offloaded I/O threshold

In reply to Chris Leech
Quoted 12 lines · Chris Leech

Any socket recv of less than this ammount will not be offloaded

---

  include/linux/sysctl.h     |    1 +
  include/net/tcp.h          |    1 +
  net/core/user_dma.c        |    4 ++++
  net/ipv4/sysctl_net_ipv4.c |   10 ++++++++++
  4 files changed, 16 insertions(+), 0 deletions(-)

Documentation/networking/ip-sysctl.txt too, please.

maintainer
#12 lore raw

Re: [PATCH 3/9] [I/OAT] Setup the networking subsystem as a DMA client

In reply to Chris Leech

From: Chris Leech <[email protected]>
Date: Tue, 23 May 2006 17:20:15 -0700

net/core/dev.c static int dev_cpu_callback(struct notif · line 3332

Quoted 6 lines
net/core/dev.c +6 -0
+static void net_dma_rebalance(void)
+{
+	unsigned int cpu, i, n;
+	struct dma_chan *chan;
+
+	lock_cpu_hotplug();

You can't call lock_cpu_hotplug(), because that sleeps and takes semaphores and we currently hold a spinlock taken here:

net/core/dev.c static int dev_cpu_callback(struct notif · line 3375

Quoted 9 lines
net/core/dev.c +9 -0
+static void netdev_dma_event(struct dma_client *client, struct dma_chan *chan,
+	enum dma_event event)
+{
+	spin_lock(&net_dma_event_lock);
+	switch (event) {
+	case DMA_RESOURCE_ADDED:
+		net_dma_count++;
+		net_dma_rebalance();
+		break;

You'll need to run this DMA rebalancing asynchronously in process context via keventd or similar to deal with this locking bug.

#3 lore raw

Re: [PATCH 1/9] [I/OAT] DMA memcpy subsystem

In reply to Andrew Morton
Quoted 7 lines · Andrew Morton

Review

Quoted 2 lines · Chris Leech
(quoted diff) +1 -0
+	for_each_cpu(i)

That's about to be deleted. Please use for_each_possible_cpu().

That's if for_each_possible_cpu() is appropriate.

It is -- those loops traverse chan->local, which is alloc_percpu'd, which allocates for all possible cpus.

#5 lore raw

Re: [PATCH 1/9] [I/OAT] DMA memcpy subsystem

In reply to Chris Leech

Hi,

drivers/dma/dmaengine.c line 403

Quoted 3 lines · Chris Leech
drivers/dma/dmaengine.c new file +3 -0
+EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
+EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
+EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);

Is there a specific reason for why you chose to export 3 different memcpu calls? They're all just wrapped to the same internals.

It would seem to make sense to have the client do their own page_address(page) + offset calculations and just export one function?

-Olof

#9 lore raw

Re: [PATCH 2/9] [I/OAT] Driver for the Intel(R) I/OAT DMA engine

In reply to Chris Leech

Hi,

Minor nitpick below:

Quoted 1 line · Chris Leech

+static int enumerate_dma_channels(struct ioat_device *device)

[...]

drivers/dma/ioatdma.c line 752

Quoted 1 line
drivers/dma/ioatdma.c new file +1 -0
+	enumerate_dma_channels(device);

Return value is never used, might as well change the function declaration to void.

-Olof

#6 lore raw

Re: [PATCH 1/9] [I/OAT] DMA memcpy subsystem

In reply to Olof Johansson
Quoted 5 lines · Olof Johansson

Is there a specific reason for why you chose to export 3 different memcpu calls? They're all just wrapped to the same internals.

It would seem to make sense to have the client do their own page_address(page) + offset calculations and just export one function?

Nevermind. I'm too used to 64-bit environments where all memory is always addressable to the kernel. There's obvious reasons to do it on 32-bit platforms to avoid the extra kernel mapping.

-Olof

#10 lore raw

Re: [PATCH 2/9] [I/OAT] Driver for the Intel(R) I/OAT DMA engine

In reply to Chris Leech
Quoted 23 lines · Chris Leech

Adds a new ioatdma driver

---

 drivers/dma/Kconfig             |    9 
 drivers/dma/Makefile            |    1 
 drivers/dma/ioatdma.c           |  839 +++++++++++++++++++++++++++++++++++++++
 drivers/dma/ioatdma.h           |  126 ++++++
 drivers/dma/ioatdma_hw.h        |   52 ++
 drivers/dma/ioatdma_io.h        |  118 +++++
 drivers/dma/ioatdma_registers.h |  126 ++++++
 7 files changed, 1271 insertions(+), 0 deletions(-)
drivers/dma/Kconfig +1 -0
index f9ac4bc..0f15e76 100644
@@ -10,4 +10,13 @@ config DMA_ENGINE
 	  DMA engines offload copy operations from the CPU to dedicated
 	  hardware, allowing the copies to happen asynchronously.
 
+comment "DMA Devices"

drivers/dma/Kconfig config DMA_ENGINE · line 22

Quoted 10 lines
drivers/dma/Kconfig +8 -0
+
+config INTEL_IOATDMA
+	tristate "Intel I/OAT DMA support"
+	depends on DMA_ENGINE && PCI
+	default m
+	---help---
+	  Enable support for the Intel(R) I/OAT DMA engine.
+
 endmenu
...

- please don't use "default m"
- can you enhance the help text?
e.g. you could list which hardware contains this DMA engine

cu
Adrian

Signature · 4 lines
       "Is there not promise of rain?" Ling Tan asked suddenly out
        of the darkness. There had been need of rain for many days.
       "Only a promise," Lao Er said.
                                       Pearl S. Buck - Dragon Seed