Skip to contentSkip to conversation

net_dma removal and a new dma-debug check

PATCH In review Dan Williams 2 participants 6 messages Dec 18, 2013

Subsystems (12)
lore
Share & tools
lore
#0 lore raw

cover net_dma removal and a new dma-debug check

NET_DMA violates the expectation of the dma-mapping api by allowing the cpu to touch receive buffer memory while dma is active. If this activity triggers a copy-on-write event the application may see missing data as the dma completes to the old mapping.

[PATCH 1/3] net_dma: mark broken
Detail the failure and provide a small one-liner for -stable

[PATCH 2/3] net_dma: remove
Mark the tcp_dma_copybreak sysctl as removed and remove the rest of the implementation

[PATCH 3/3] dma debug: flag copy-on-write events to pages undergoing dma Extend dma-debug to catch this condition in the future.

Looking to take this through the dmaengine tree with a netdev ack.

---

 Documentation/ABI/removed/net_dma      |    8 +
 Documentation/networking/ip-sysctl.txt |    6 -
 drivers/dma/Kconfig                    |   11 -
 drivers/dma/Makefile                   |    1 
 drivers/dma/dmaengine.c                |  104 ------------
 drivers/dma/ioat/dma.c                 |    1 
 drivers/dma/ioat/dma.h                 |    7 -
 drivers/dma/ioat/dma_v2.c              |    1 
 drivers/dma/ioat/dma_v3.c              |    1 
 drivers/dma/iovlock.c                  |  280 --------------------------------
 include/linux/dma-debug.h              |    6 +
 include/linux/dmaengine.h              |   22 ---
 include/linux/skbuff.h                 |    7 -
 include/linux/tcp.h                    |    7 -
 include/net/netdma.h                   |   32 ----
 include/net/sock.h                     |   18 --
 include/net/tcp.h                      |    7 -
 kernel/sysctl_binary.c                 |    1 
 lib/dma-debug.c                        |  130 +++++++++++++--
 mm/memory.c                            |    3 
 net/core/Makefile                      |    1 
 net/core/dev.c                         |   10 -
 net/core/sock.c                        |    6 -
 net/core/user_dma.c                    |  131 ---------------
 net/dccp/proto.c                       |    4 
 net/ipv4/sysctl_net_ipv4.c             |    9 -
 net/ipv4/tcp.c                         |  147 ++---------------
 net/ipv4/tcp_input.c                   |   61 -------
 net/ipv4/tcp_ipv4.c                    |   18 --
 net/ipv6/tcp_ipv6.c                    |   13 -
 net/llc/af_llc.c                       |   10 +
 31 files changed, 161 insertions(+), 902 deletions(-)
 create mode 100644 Documentation/ABI/removed/net_dma
 delete mode 100644 drivers/dma/iovlock.c
 delete mode 100644 include/net/netdma.h
 delete mode 100644 net/core/user_dma.c
#1 lore raw

PATCH 1/3 net_dma: mark broken

net_dma can cause data to be copied to a stale mapping if a copy-on-write fault occurs during dma. The application sees missing data.

The following trace is triggered by modifying the kernel to WARN if it ever triggers copy-on-write on a page that is undergoing dma:

 WARNING: CPU: 24 PID: 2529 at lib/dma-debug.c:485 debug_dma_assert_idle+0xd2/0x120()
 ioatdma 0000:00:04.0: DMA-API: cpu touching an active dma mapped page [pfn=0x16bcd9]
 Modules linked in: iTCO_wdt iTCO_vendor_support ioatdma lpc_ich pcspkr dca
 CPU: 24 PID: 2529 Comm: linbug Tainted: G        W    3.13.0-rc1+ #353
  00000000000001e5 ffff88016f45f688 ffffffff81751041 ffff88017ab0ef70
  ffff88016f45f6d8 ffff88016f45f6c8 ffffffff8104ed9c ffffffff810f3646
  ffff8801768f4840 0000000000000282 ffff88016f6cca10 00007fa2bb699349
 Call Trace:
  [<ffffffff81751041>] dump_stack+0x46/0x58
  [<ffffffff8104ed9c>] warn_slowpath_common+0x8c/0xc0
  [<ffffffff810f3646>] ? ftrace_pid_func+0x26/0x30
  [<ffffffff8104ee86>] warn_slowpath_fmt+0x46/0x50
  [<ffffffff8139c062>] debug_dma_assert_idle+0xd2/0x120
  [<ffffffff81154a40>] do_wp_page+0xd0/0x790
  [<ffffffff811582ac>] handle_mm_fault+0x51c/0xde0
  [<ffffffff813830b9>] ? copy_user_enhanced_fast_string+0x9/0x20
  [<ffffffff8175fc2c>] __do_page_fault+0x19c/0x530
  [<ffffffff8175c196>] ? _raw_spin_lock_bh+0x16/0x40
  [<ffffffff810f3539>] ? trace_clock_local+0x9/0x10
  [<ffffffff810fa1f4>] ? rb_reserve_next_event+0x64/0x310
  [<ffffffffa0014c00>] ? ioat2_dma_prep_memcpy_lock+0x60/0x130 [ioatdma]
  [<ffffffff8175ffce>] do_page_fault+0xe/0x10
  [<ffffffff8175c862>] page_fault+0x22/0x30
  [<ffffffff81643991>] ? __kfree_skb+0x51/0xd0
  [<ffffffff813830b9>] ? copy_user_enhanced_fast_string+0x9/0x20
  [<ffffffff81388ea2>] ? memcpy_toiovec+0x52/0xa0
  [<ffffffff8164770f>] skb_copy_datagram_iovec+0x5f/0x2a0
  [<ffffffff8169d0f4>] tcp_rcv_established+0x674/0x7f0
  [<ffffffff816a68c5>] tcp_v4_do_rcv+0x2e5/0x4a0
  [..]
 ---[ end trace e30e3b01191b7617 ]---
 Mapped at:
  [<ffffffff8139c169>] debug_dma_map_page+0xb9/0x160
  [<ffffffff8142bf47>] dma_async_memcpy_pg_to_pg+0x127/0x210
  [<ffffffff8142cce9>] dma_memcpy_pg_to_iovec+0x119/0x1f0
  [<ffffffff81669d3c>] dma_skb_copy_datagram_iovec+0x11c/0x2b0
  [<ffffffff8169d1ca>] tcp_rcv_established+0x74a/0x7f0:

...the problem is that the receive path falls back to cpu-copy in several locations and this trace is just one of the areas. A few options were considered to fix this:

1/ sync all dma whenever a cpu copy branch is taken

2/ modify the page fault handler to hold off while dma is in-flight

Option 1 adds yet more cpu overhead to an "offload" that struggles to compete with cpu-copy. Option 2 adds checks for behavior that is already documented as broken when using get_user_pages(). At a minimum a debug mode is warranted to catch and flag these violations of the dma-api vs get_user_pages().

Thanks to David for his reproducer.

---
 drivers/dma/Kconfig |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)
drivers/dma/Kconfig +1 -0
index 132a4fd375b2..c823daaf9043 100644
@@ -355,6 +355,7 @@ config NET_DMA
 	bool "Network: TCP receive copy offload"
 	depends on DMA_ENGINE && NET
 	default (INTEL_IOATDMA || FSL_DMA)
+	depends on BROKEN
 	help
 	  This enables the use of DMA engines in the network stack to
 	  offload receive copy-to-user operations, freeing CPU cycles.
#2 lore raw

Re: [PATCH 1/3] net_dma: mark broken

[ adding davem directly to cc ]

Dave, looking for an ack to take this through the dmaengine tree, and wondering if you want me to take patch 2 through dmaengine as well:

http://marc.info/?l=linux-netdev&m=138732577614066&w=2

Quoted 83 lines · Dan Williams

net_dma can cause data to be copied to a stale mapping if a copy-on-write fault occurs during dma. The application sees missing data.

The following trace is triggered by modifying the kernel to WARN if it ever triggers copy-on-write on a page that is undergoing dma:

 WARNING: CPU: 24 PID: 2529 at lib/dma-debug.c:485 debug_dma_assert_idle+0xd2/0x120()
 ioatdma 0000:00:04.0: DMA-API: cpu touching an active dma mapped page [pfn=0x16bcd9]
 Modules linked in: iTCO_wdt iTCO_vendor_support ioatdma lpc_ich pcspkr dca
 CPU: 24 PID: 2529 Comm: linbug Tainted: G        W    3.13.0-rc1+ #353
  00000000000001e5 ffff88016f45f688 ffffffff81751041 ffff88017ab0ef70
  ffff88016f45f6d8 ffff88016f45f6c8 ffffffff8104ed9c ffffffff810f3646
  ffff8801768f4840 0000000000000282 ffff88016f6cca10 00007fa2bb699349
 Call Trace:
  [<ffffffff81751041>] dump_stack+0x46/0x58
  [<ffffffff8104ed9c>] warn_slowpath_common+0x8c/0xc0
  [<ffffffff810f3646>] ? ftrace_pid_func+0x26/0x30
  [<ffffffff8104ee86>] warn_slowpath_fmt+0x46/0x50
  [<ffffffff8139c062>] debug_dma_assert_idle+0xd2/0x120
  [<ffffffff81154a40>] do_wp_page+0xd0/0x790
  [<ffffffff811582ac>] handle_mm_fault+0x51c/0xde0
  [<ffffffff813830b9>] ? copy_user_enhanced_fast_string+0x9/0x20
  [<ffffffff8175fc2c>] __do_page_fault+0x19c/0x530
  [<ffffffff8175c196>] ? _raw_spin_lock_bh+0x16/0x40
  [<ffffffff810f3539>] ? trace_clock_local+0x9/0x10
  [<ffffffff810fa1f4>] ? rb_reserve_next_event+0x64/0x310
  [<ffffffffa0014c00>] ? ioat2_dma_prep_memcpy_lock+0x60/0x130 [ioatdma]
  [<ffffffff8175ffce>] do_page_fault+0xe/0x10
  [<ffffffff8175c862>] page_fault+0x22/0x30
  [<ffffffff81643991>] ? __kfree_skb+0x51/0xd0
  [<ffffffff813830b9>] ? copy_user_enhanced_fast_string+0x9/0x20
  [<ffffffff81388ea2>] ? memcpy_toiovec+0x52/0xa0
  [<ffffffff8164770f>] skb_copy_datagram_iovec+0x5f/0x2a0
  [<ffffffff8169d0f4>] tcp_rcv_established+0x674/0x7f0
  [<ffffffff816a68c5>] tcp_v4_do_rcv+0x2e5/0x4a0
  [..]
 ---[ end trace e30e3b01191b7617 ]---
 Mapped at:
  [<ffffffff8139c169>] debug_dma_map_page+0xb9/0x160
  [<ffffffff8142bf47>] dma_async_memcpy_pg_to_pg+0x127/0x210
  [<ffffffff8142cce9>] dma_memcpy_pg_to_iovec+0x119/0x1f0
  [<ffffffff81669d3c>] dma_skb_copy_datagram_iovec+0x11c/0x2b0
  [<ffffffff8169d1ca>] tcp_rcv_established+0x74a/0x7f0:

...the problem is that the receive path falls back to cpu-copy in several locations and this trace is just one of the areas. A few options were considered to fix this:

1/ sync all dma whenever a cpu copy branch is taken

2/ modify the page fault handler to hold off while dma is in-flight

Option 1 adds yet more cpu overhead to an "offload" that struggles to compete with cpu-copy. Option 2 adds checks for behavior that is already documented as broken when using get_user_pages(). At a minimum a debug mode is warranted to catch and flag these violations of the dma-api vs get_user_pages().

Thanks to David for his reproducer.

---
 drivers/dma/Kconfig |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)
drivers/dma/Kconfig +1 -0
index 132a4fd375b2..c823daaf9043 100644
@@ -355,6 +355,7 @@ config NET_DMA
        bool "Network: TCP receive copy offload"
        depends on DMA_ENGINE && NET
        default (INTEL_IOATDMA || FSL_DMA)
+       depends on BROKEN
        help
          This enables the use of DMA engines in the network stack to
          offload receive copy-to-user operations, freeing CPU cycles.
maintainer Acked-by
#3 lore raw

Re: [PATCH 1/3] net_dma: mark broken

From: Dan Williams <[email protected]>
Date: Wed, 18 Dec 2013 10:05:51 -0800

Quoted 4 lines

[ adding davem directly to cc ]

Dave, looking for an ack to take this through the dmaengine tree, and wondering if you want me to take patch 2 through dmaengine as well:

Please take it all:

#4 lore raw

PATCH 2/3 net_dma: remove

In reply to Dan Williams

Per commit "net_dma: mark broken" net_dma is no longer used and there is no plan to fix it.

Cc Dave Jiang <[email protected]>
Cc Vinod Koul <[email protected]>
Cc David Whipple <[email protected]>
Cc Alexander Duyck <[email protected]>
---
 Documentation/ABI/removed/net_dma      |    8 +
 Documentation/networking/ip-sysctl.txt |    6 -
 drivers/dma/Kconfig                    |   12 -
 drivers/dma/Makefile                   |    1 
 drivers/dma/dmaengine.c                |  104 ------------
 drivers/dma/ioat/dma.c                 |    1 
 drivers/dma/ioat/dma.h                 |    7 -
 drivers/dma/ioat/dma_v2.c              |    1 
 drivers/dma/ioat/dma_v3.c              |    1 
 drivers/dma/iovlock.c                  |  280 --------------------------------
 include/linux/dmaengine.h              |   22 ---
 include/linux/skbuff.h                 |    7 -
 include/linux/tcp.h                    |    7 -
 include/net/netdma.h                   |   32 ----
 include/net/sock.h                     |   18 --
 include/net/tcp.h                      |    7 -
 kernel/sysctl_binary.c                 |    1 
 net/core/Makefile                      |    1 
 net/core/dev.c                         |   10 -
 net/core/sock.c                        |    6 -
 net/core/user_dma.c                    |  131 ---------------
 net/dccp/proto.c                       |    4 
 net/ipv4/sysctl_net_ipv4.c             |    9 -
 net/ipv4/tcp.c                         |  147 ++---------------
 net/ipv4/tcp_input.c                   |   61 -------
 net/ipv4/tcp_ipv4.c                    |   18 --
 net/ipv6/tcp_ipv6.c                    |   13 -
 net/llc/af_llc.c                       |   10 +
 28 files changed, 35 insertions(+), 890 deletions(-)
 create mode 100644 Documentation/ABI/removed/net_dma
 delete mode 100644 drivers/dma/iovlock.c
 delete mode 100644 include/net/netdma.h
 delete mode 100644 net/core/user_dma.c
Documentation/ABI/removed/net_dma new file +8 -0
new file mode 100644
index 000000000000..a173aecc2f18
@@ -0,0 +1,8 @@
+What:		tcp_dma_copybreak sysctl
+Date:		Removed in kernel v3.13
+Contact:	Dan Williams <[email protected]>
+Description:
+	Formerly the lower limit, in bytes, of the size of socket reads
+	that will be offloaded to a DMA copy engine.  Removed due to
+	coherency issues of the cpu potentially touching the buffers
+	while dma is in flight.
Documentation/networking/ip-sysctl.txt +0 -6
index 3c12d9a7ed00..bdd8a67f0be2 100644
@@ -538,12 +538,6 @@ tcp_workaround_signed_windows - BOOLEAN
 	not receive a window scaling option from them.
 	Default: 0
 
-tcp_dma_copybreak - INTEGER
-	Lower limit, in bytes, of the size of socket reads that will be
-	offloaded to a DMA copy engine, if one is present in the system
-	and CONFIG_NET_DMA is enabled.
-	Default: 4096
-
 tcp_thin_linear_timeouts - BOOLEAN
 	Enable dynamic triggering of linear timeouts for thin streams.
 	If set, a check is performed upon retransmission by timeout to
drivers/dma/Kconfig +0 -12
index c823daaf9043..b24f13195272 100644
@@ -351,18 +351,6 @@ config DMA_OF
 comment "DMA Clients"
 	depends on DMA_ENGINE
 
-config NET_DMA
-	bool "Network: TCP receive copy offload"
-	depends on DMA_ENGINE && NET
-	default (INTEL_IOATDMA || FSL_DMA)
-	depends on BROKEN
-	help
-	  This enables the use of DMA engines in the network stack to
-	  offload receive copy-to-user operations, freeing CPU cycles.
-
-	  Say Y here if you enabled INTEL_IOATDMA or FSL_DMA, otherwise
-	  say N.
-
 config ASYNC_TX_DMA
 	bool "Async_tx: Offload support for the async_tx api"
 	depends on DMA_ENGINE
drivers/dma/Makefile +0 -1
index 0ce2da97e429..024b008a25de 100644
@@ -6,7 +6,6 @@ obj-$(CONFIG_DMA_VIRTUAL_CHANNELS) += virt-dma.o
 obj-$(CONFIG_DMA_ACPI) += acpi-dma.o
 obj-$(CONFIG_DMA_OF) += of-dma.o
 
-obj-$(CONFIG_NET_DMA) += iovlock.o
 obj-$(CONFIG_INTEL_MID_DMAC) += intel_mid_dma.o
 obj-$(CONFIG_DMATEST) += dmatest.o
 obj-$(CONFIG_INTEL_IOATDMA) += ioat/
drivers/dma/dmaengine.c +0 -104

110 lines not on this page · raw message

drivers/dma/ioat/dma.c +0 -1
index 1a49c777607c..97fa394ca855 100644
@@ -1175,7 +1175,6 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
 	err = ioat_probe(device);
 	if (err)
 		return err;
-	ioat_set_tcp_copy_break(4096);
 	err = ioat_register(device);
 	if (err)
 		return err;
drivers/dma/ioat/dma.h +0 -7
index 11fb877ddca9..664ec9cbd651 100644
@@ -214,13 +214,6 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
 #define dump_desc_dbg(c, d) \
 	({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; })
 
-static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
-{
-	#ifdef CONFIG_NET_DMA
-	sysctl_tcp_dma_copybreak = copybreak;
-	#endif
-}
-
 static inline struct ioat_chan_common *
 ioat_chan_by_index(struct ioatdma_device *device, int index)
 {
drivers/dma/ioat/dma_v2.c +0 -1
index 5d3affe7e976..31e8098e444f 100644
@@ -900,7 +900,6 @@ int ioat2_dma_probe(struct ioatdma_device *device, int dca)
 	err = ioat_probe(device);
 	if (err)
 		return err;
-	ioat_set_tcp_copy_break(2048);
 
 	list_for_each_entry(c, &dma->channels, device_node) {
 		chan = to_chan_common(c);
drivers/dma/ioat/dma_v3.c +0 -1
index 820817e97e62..4bb81346bee2 100644
@@ -1652,7 +1652,6 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
 	err = ioat_probe(device);
 	if (err)
 		return err;
-	ioat_set_tcp_copy_break(262144);
 
 	list_for_each_entry(c, &dma->channels, device_node) {
 		chan = to_chan_common(c);
drivers/dma/iovlock.c deleted +0 -280

280 lines not on this page · raw message

include/linux/dmaengine.h +1 -21
index 41cf0c399288..890545871af0 100644
@@ -875,18 +875,6 @@ static inline void dmaengine_put(void)
 }
 #endif
 
-#ifdef CONFIG_NET_DMA
-#define net_dmaengine_get()	dmaengine_get()
-#define net_dmaengine_put()	dmaengine_put()
-#else
-static inline void net_dmaengine_get(void)
-{
-}
-static inline void net_dmaengine_put(void)
-{
-}
-#endif
-
 #ifdef CONFIG_ASYNC_TX_DMA
 #define async_dmaengine_get()	dmaengine_get()
 #define async_dmaengine_put()	dmaengine_put()
@@ -908,16 +896,8 @@ async_dma_find_channel(enum dma_transaction_type type)
 	return NULL;
 }
 #endif /* CONFIG_ASYNC_TX_DMA */
-
-dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
-	void *dest, void *src, size_t len);
-dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
-	struct page *page, unsigned int offset, void *kdata, size_t len);
-dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
-	struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
-	unsigned int src_off, size_t len);
 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
-	struct dma_chan *chan);
+				  struct dma_chan *chan);
 
 static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
 {
include/linux/skbuff.h +2 -5
index bec1cc7d5e3c..457bea8a4e4d 100644
@@ -496,11 +496,8 @@ struct sk_buff {
 	/* 6/8 bit hole (depending on ndisc_nodetype presence) */
 	kmemcheck_bitfield_end(flags2);
 
-#if defined CONFIG_NET_DMA || defined CONFIG_NET_RX_BUSY_POLL
-	union {
-		unsigned int	napi_id;
-		dma_cookie_t	dma_cookie;
-	};
+#ifdef CONFIG_NET_RX_BUSY_POLL
+	unsigned int	napi_id;
 #endif
 #ifdef CONFIG_NETWORK_SECMARK
 	__u32			secmark;
include/linux/tcp.h +0 -7
index d68633452d9b..90b88d229e03 100644
@@ -169,13 +169,6 @@ struct tcp_sock {
 		struct iovec		*iov;
 		int			memory;
 		int			len;
-#ifdef CONFIG_NET_DMA
-		/* members for async copy */
-		struct dma_chan		*dma_chan;
-		int			wakeup;
-		struct dma_pinned_list	*pinned_list;
-		dma_cookie_t		dma_cookie;
-#endif
 	} ucopy;
 
 	u32	snd_wl1;	/* Sequence for window update		*/
include/net/netdma.h deleted +0 -32
deleted file mode 100644
index 8ba8ce284eeb..000000000000
@@ -1,32 +0,0 @@
-/*
- * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- *
- * The full GNU General Public License is included in this distribution in the
- * file called COPYING.
- */
-#ifndef NETDMA_H
-#define NETDMA_H
-#ifdef CONFIG_NET_DMA
-#include <linux/dmaengine.h>
-#include <linux/skbuff.h>
-
-int dma_skb_copy_datagram_iovec(struct dma_chan* chan,
-		struct sk_buff *skb, int offset, struct iovec *to,
-		size_t len, struct dma_pinned_list *pinned_list);
-
-#endif /* CONFIG_NET_DMA */
-#endif /* NETDMA_H */
include/net/sock.h +1 -17
index e3a18ff0c38b..49c0570c5ebc 100644
@@ -354,10 +354,6 @@ struct sock {
 	struct sk_filter __rcu	*sk_filter;
 	struct socket_wq __rcu	*sk_wq;
 
-#ifdef CONFIG_NET_DMA
-	struct sk_buff_head	sk_async_wait_queue;
-#endif
-
 #ifdef CONFIG_XFRM
 	struct xfrm_policy	*sk_policy[2];
 #endif
@@ -2200,27 +2196,15 @@ void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
  * sk_eat_skb - Release a skb if it is no longer needed
  * @sk: socket to eat this skb from
  * @skb: socket buffer to eat
- * @copied_early: flag indicating whether DMA operations copied this data early
  *
  * This routine must be called with interrupts disabled or with the socket
  * locked so that the sk_buff queue operation is ok.
 */
-#ifdef CONFIG_NET_DMA
-static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
-{
-	__skb_unlink(skb, &sk->sk_receive_queue);
-	if (!copied_early)
-		__kfree_skb(skb);
-	else
-		__skb_queue_tail(&sk->sk_async_wait_queue, skb);
-}
-#else
-static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
+static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
 {
 	__skb_unlink(skb, &sk->sk_receive_queue);
 	__kfree_skb(skb);
 }
-#endif
 
 static inline
 struct net *sock_net(const struct sock *sk)
include/net/tcp.h +0 -7
index 70e55d200610..d44048f8831a 100644
@@ -267,7 +267,6 @@ extern int sysctl_tcp_adv_win_scale;
 extern int sysctl_tcp_tw_reuse;
 extern int sysctl_tcp_frto;
 extern int sysctl_tcp_low_latency;
-extern int sysctl_tcp_dma_copybreak;
 extern int sysctl_tcp_nometrics_save;
 extern int sysctl_tcp_moderate_rcvbuf;
 extern int sysctl_tcp_tso_win_divisor;
@@ -1032,12 +1031,6 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
 	tp->ucopy.len = 0;
 	tp->ucopy.memory = 0;
 	skb_queue_head_init(&tp->ucopy.prequeue);
-#ifdef CONFIG_NET_DMA
-	tp->ucopy.dma_chan = NULL;
-	tp->ucopy.wakeup = 0;
-	tp->ucopy.pinned_list = NULL;
-	tp->ucopy.dma_cookie = 0;
-#endif
 }
 
 bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
kernel/sysctl_binary.c +0 -1
index 653cbbd9e7ad..d457005acedf 100644
@@ -390,7 +390,6 @@ static const struct bin_table bin_net_ipv4_table[] = {
 	{ CTL_INT,	NET_TCP_MTU_PROBING,			"tcp_mtu_probing" },
 	{ CTL_INT,	NET_TCP_BASE_MSS,			"tcp_base_mss" },
 	{ CTL_INT,	NET_IPV4_TCP_WORKAROUND_SIGNED_WINDOWS,	"tcp_workaround_signed_windows" },
-	{ CTL_INT,	NET_TCP_DMA_COPYBREAK,			"tcp_dma_copybreak" },
 	{ CTL_INT,	NET_TCP_SLOW_START_AFTER_IDLE,		"tcp_slow_start_after_idle" },
 	{ CTL_INT,	NET_CIPSOV4_CACHE_ENABLE,		"cipso_cache_enable" },
 	{ CTL_INT,	NET_CIPSOV4_CACHE_BUCKET_SIZE,		"cipso_cache_bucket_size" },
net/core/Makefile +0 -1
index b33b996f5dd6..5f98e5983bd3 100644
@@ -16,7 +16,6 @@ obj-y += net-sysfs.o
 obj-$(CONFIG_PROC_FS) += net-procfs.o
 obj-$(CONFIG_NET_PKTGEN) += pktgen.o
 obj-$(CONFIG_NETPOLL) += netpoll.o
-obj-$(CONFIG_NET_DMA) += user_dma.o
 obj-$(CONFIG_FIB_RULES) += fib_rules.o
 obj-$(CONFIG_TRACEPOINTS) += net-traces.o
 obj-$(CONFIG_NET_DROP_MONITOR) += drop_monitor.o
net/core/dev.c +0 -10
index ba3b7ea5ebb3..677a5a4dcca7 100644
@@ -1262,7 +1262,6 @@ static int __dev_open(struct net_device *dev)
 		clear_bit(__LINK_STATE_START, &dev->state);
 	else {
 		dev->flags |= IFF_UP;
-		net_dmaengine_get();
 		dev_set_rx_mode(dev);
 		dev_activate(dev);
 		add_device_randomness(dev->dev_addr, dev->addr_len);
@@ -1338,7 +1337,6 @@ static int __dev_close_many(struct list_head *head)
 			ops->ndo_stop(dev);
 
 		dev->flags &= ~IFF_UP;
-		net_dmaengine_put();
 	}
 
 	return 0;
@@ -4362,14 +4360,6 @@ static void net_rx_action(struct softirq_action *h)
 out:
 	net_rps_action_and_irq_enable(sd);
 
-#ifdef CONFIG_NET_DMA
-	/*
-	 * There may not be any more sk_buffs coming right now, so push
-	 * any pending DMA copies to hardware
-	 */
-	dma_issue_pending_all();
-#endif
-
 	return;
 
 softnet_break:
net/core/sock.c +0 -6
index ab20ed9b0f31..411dab3a5726 100644
@@ -1461,9 +1461,6 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
 		atomic_set(&newsk->sk_omem_alloc, 0);
 		skb_queue_head_init(&newsk->sk_receive_queue);
 		skb_queue_head_init(&newsk->sk_write_queue);
-#ifdef CONFIG_NET_DMA
-		skb_queue_head_init(&newsk->sk_async_wait_queue);
-#endif
 
 		spin_lock_init(&newsk->sk_dst_lock);
 		rwlock_init(&newsk->sk_callback_lock);
@@ -2290,9 +2287,6 @@ void sock_init_data(struct socket *sock, struct sock *sk)
 	skb_queue_head_init(&sk->sk_receive_queue);
 	skb_queue_head_init(&sk->sk_write_queue);
 	skb_queue_head_init(&sk->sk_error_queue);
-#ifdef CONFIG_NET_DMA
-	skb_queue_head_init(&sk->sk_async_wait_queue);
-#endif
 
 	sk->sk_send_head	=	NULL;
 
net/core/user_dma.c deleted +0 -131

131 lines not on this page · raw message

net/dccp/proto.c +2 -2
index eb892b4f4814..f9076f295b13 100644
@@ -848,7 +848,7 @@ int dccp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 		default:
 			dccp_pr_debug("packet_type=%s\n",
 				      dccp_packet_name(dh->dccph_type));
-			sk_eat_skb(sk, skb, false);
+			sk_eat_skb(sk, skb);
 		}
 verify_sock_status:
 		if (sock_flag(sk, SOCK_DONE)) {
@@ -905,7 +905,7 @@ verify_sock_status:
 			len = skb->len;
 	found_fin_ok:
 		if (!(flags & MSG_PEEK))
-			sk_eat_skb(sk, skb, false);
+			sk_eat_skb(sk, skb);
 		break;
 	} while (1);
 out:
net/ipv4/sysctl_net_ipv4.c +0 -9
index 3d69ec8dac57..79a90b92e12d 100644
@@ -642,15 +642,6 @@ static struct ctl_table ipv4_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec
 	},
-#ifdef CONFIG_NET_DMA
-	{
-		.procname	= "tcp_dma_copybreak",
-		.data		= &sysctl_tcp_dma_copybreak,
-		.maxlen		= sizeof(int),
-		.mode		= 0644,
-		.proc_handler	= proc_dointvec
-	},
-#endif
 	{
 		.procname	= "tcp_slow_start_after_idle",
 		.data		= &sysctl_tcp_slow_start_after_idle,
net/ipv4/tcp.c +14 -133

227 lines not on this page · raw message

net/ipv4/tcp_input.c +0 -61

85 lines not on this page · raw message

net/ipv4/tcp_ipv4.c +1 -17
index 59a6f8b90cd9..dc92ba9d0350 100644
@@ -72,7 +72,6 @@
 #include <net/inet_common.h>
 #include <net/timewait_sock.h>
 #include <net/xfrm.h>
-#include <net/netdma.h>
 #include <net/secure_seq.h>
 #include <net/tcp_memcontrol.h>
 #include <net/busy_poll.h>
@@ -2000,18 +1999,8 @@ process:
 	bh_lock_sock_nested(sk);
 	ret = 0;
 	if (!sock_owned_by_user(sk)) {
-#ifdef CONFIG_NET_DMA
-		struct tcp_sock *tp = tcp_sk(sk);
-		if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
-			tp->ucopy.dma_chan = net_dma_find_channel();
-		if (tp->ucopy.dma_chan)
+		if (!tcp_prequeue(sk, skb))
 			ret = tcp_v4_do_rcv(sk, skb);
-		else
-#endif
-		{
-			if (!tcp_prequeue(sk, skb))
-				ret = tcp_v4_do_rcv(sk, skb);
-		}
 	} else if (unlikely(sk_add_backlog(sk, skb,
 					   sk->sk_rcvbuf + sk->sk_sndbuf))) {
 		bh_unlock_sock(sk);
@@ -2170,11 +2159,6 @@ void tcp_v4_destroy_sock(struct sock *sk)
 	}
 #endif
 
-#ifdef CONFIG_NET_DMA
-	/* Cleans up our sk_async_wait_queue */
-	__skb_queue_purge(&sk->sk_async_wait_queue);
-#endif
-
 	/* Clean prequeue, it must be empty really */
 	__skb_queue_purge(&tp->ucopy.prequeue);
 
net/ipv6/tcp_ipv6.c +1 -12
index 0740f93a114a..e27972590379 100644
@@ -59,7 +59,6 @@
 #include <net/snmp.h>
 #include <net/dsfield.h>
 #include <net/timewait_sock.h>
-#include <net/netdma.h>
 #include <net/inet_common.h>
 #include <net/secure_seq.h>
 #include <net/tcp_memcontrol.h>
@@ -1504,18 +1503,8 @@ process:
 	bh_lock_sock_nested(sk);
 	ret = 0;
 	if (!sock_owned_by_user(sk)) {
-#ifdef CONFIG_NET_DMA
-		struct tcp_sock *tp = tcp_sk(sk);
-		if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
-			tp->ucopy.dma_chan = net_dma_find_channel();
-		if (tp->ucopy.dma_chan)
+		if (!tcp_prequeue(sk, skb))
 			ret = tcp_v6_do_rcv(sk, skb);
-		else
-#endif
-		{
-			if (!tcp_prequeue(sk, skb))
-				ret = tcp_v6_do_rcv(sk, skb);
-		}
 	} else if (unlikely(sk_add_backlog(sk, skb,
 					   sk->sk_rcvbuf + sk->sk_sndbuf))) {
 		bh_unlock_sock(sk);
net/llc/af_llc.c +5 -5
index 7b01b9f5846c..e1b46709f8d6 100644
@@ -838,7 +838,7 @@ static int llc_ui_recvmsg(struct kiocb *iocb, struct socket *sock,
 
 		if (!(flags & MSG_PEEK)) {
 			spin_lock_irqsave(&sk->sk_receive_queue.lock, cpu_flags);
-			sk_eat_skb(sk, skb, false);
+			sk_eat_skb(sk, skb);
 			spin_unlock_irqrestore(&sk->sk_receive_queue.lock, cpu_flags);
 			*seq = 0;
 		}
@@ -860,10 +860,10 @@ copy_uaddr:
 		llc_cmsg_rcv(msg, skb);
 
 	if (!(flags & MSG_PEEK)) {
-			spin_lock_irqsave(&sk->sk_receive_queue.lock, cpu_flags);
-			sk_eat_skb(sk, skb, false);
-			spin_unlock_irqrestore(&sk->sk_receive_queue.lock, cpu_flags);
-			*seq = 0;
+		spin_lock_irqsave(&sk->sk_receive_queue.lock, cpu_flags);
+		sk_eat_skb(sk, skb);
+		spin_unlock_irqrestore(&sk->sk_receive_queue.lock, cpu_flags);
+		*seq = 0;
 	}
 
 	goto out;
#5 lore raw

PATCH 3/3 dma debug: flag copy-on-write events to pages undergoing dma

In reply to Dan Williams

Record actively mapped pages and provide an api for other parts of the kernel to check that a given page is dma inactive before taking action. Verified to catch the misuse of the dma-api in the NET_DMA implementation.

Cc Vinod Koul <[email protected]>
Cc Andrew Morton <[email protected]>
---
 include/linux/dma-debug.h |    6 ++
 lib/dma-debug.c           |  130 +++++++++++++++++++++++++++++++++++++++++----
 mm/memory.c               |    3 +
 3 files changed, 126 insertions(+), 13 deletions(-)
include/linux/dma-debug.h +6 -0
index fc0e34ce038f..fe8cb610deac 100644
@@ -85,6 +85,8 @@ extern void debug_dma_sync_sg_for_device(struct device *dev,
 
 extern void debug_dma_dump_mappings(struct device *dev);
 
+extern void debug_dma_assert_idle(struct page *page);
+
 #else /* CONFIG_DMA_API_DEBUG */
 
 static inline void dma_debug_add_bus(struct bus_type *bus)
@@ -183,6 +185,10 @@ static inline void debug_dma_dump_mappings(struct device *dev)
 {
 }
 
+static inline void debug_dma_assert_idle(struct page *page)
+{
+}
+
 #endif /* CONFIG_DMA_API_DEBUG */
 
 #endif /* __DMA_DEBUG_H */
lib/dma-debug.c +117 -13

225 lines not on this page · raw message

mm/memory.c +3 -0
index 5d9025f3b3e1..c89788436f81 100644
@@ -59,6 +59,7 @@
 #include <linux/gfp.h>
 #include <linux/migrate.h>
 #include <linux/string.h>
+#include <linux/dma-debug.h>
 
 #include <asm/io.h>
 #include <asm/pgalloc.h>
@@ -2559,6 +2560,8 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
 
 static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
 {
+	debug_dma_assert_idle(src);
+
 	/*
 	 * If the source page was a PFN mapping, we don't have
 	 * a "struct page" for it. We do a best-effort copy by