Skip to contentSkip to conversation

swiotlb: avoid double copy with swiotlb on tx socket

PATCH In review Luigi Rizzo 11 participants 30 messages Jun 15 – Aug 26

Subsystems (11)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 swiotlb: avoid double copy with swiotlb on tx socket

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent
(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

---
 drivers/base/core.c        |   1 +
 include/linux/netdevice.h  |  22 ++++
 include/linux/page-flags.h |   4 +
 include/linux/skbuff.h     |   7 +-
 include/linux/swiotlb.h    |  74 ++++++++++++
 include/net/sock.h         |  29 +++++
 kernel/dma/swiotlb.c       | 227 +++++++++++++++++++++++++++++++++++++
 mm/page_alloc.c            |  32 ++++++
 net/core/sock.c            |  98 ++++++++++++++--
 9 files changed, 485 insertions(+), 9 deletions(-)
drivers/base/core.c +1 -0
index bd2ddf2aab505..e1257dea37ba0 100644
@@ -3855,6 +3855,7 @@ void device_del(struct device *dev)
 	unsigned int noio_flag;
 
 	device_lock(dev);
+	swiotlb_device_deleted();
 	kill_device(dev);
 	device_unlock(dev);
 
include/linux/netdevice.h +22 -0
index 0e1e581efc5ac..d7e5929e73c92 100644
@@ -5368,13 +5368,35 @@ static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
 	return ops->ndo_start_xmit(skb, dev);
 }
 
+struct sock;
+
+#ifdef CONFIG_SWIOTLB
+/* Per-CPU pointer to the socket currently performing transmission.
+ * Used to bridge the networking and DMA layers, allowing the dma_map_page()
+ * path to identify the socket originating the packet and apply SWIOTLB optimizations.
+ */
+DECLARE_PER_CPU(struct sock *, current_tx_socket);
+static inline struct sock *__set_current_tx_socket(struct sock *sk)
+{
+	struct sock *old_sk = this_cpu_read(current_tx_socket);
+
+	this_cpu_write(current_tx_socket, sk);
+	return old_sk;
+}
+#else
+static inline struct sock *__set_current_tx_socket(struct sock *sk) { return NULL; }
+#endif
+
 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
 					    struct netdev_queue *txq, bool more)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
+	struct sock *old_sk;
 	netdev_tx_t rc;
 
+	old_sk = __set_current_tx_socket(skb->sk);
 	rc = __netdev_start_xmit(ops, skb, dev, more);
+	__set_current_tx_socket(old_sk);
 	if (rc == NETDEV_TX_OK)
 		txq_trans_update(dev, txq);
 
include/linux/page-flags.h +4 -0
index 7223f6f4e2b40..0ecbb404038a0 100644
@@ -923,6 +923,7 @@ enum pagetype {
 	PGTY_zsmalloc		= 0xf6,
 	PGTY_unaccepted		= 0xf7,
 	PGTY_large_kmalloc	= 0xf8,
+	PGTY_zcswiotlb		= 0xf9,
 
 	PGTY_mapcount_underflow = 0xff
 };
@@ -1055,6 +1056,9 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
 PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
 
+/* Pages in socket buffers from the swiotlb pool. */
+PAGE_TYPE_OPS(ZCSwiotlb, zcswiotlb, zcswiotlb)
+
 /**
  * PageHuge - Determine if the page belongs to hugetlbfs
  * @page: The page to test.
include/linux/skbuff.h +6 -1
index 3f06254ab1b72..62340909409e5 100644
@@ -3787,7 +3787,12 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto,
 	fragto->netmem = fragfrom->netmem;
 }
 
-bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
+/* zerocopy swiotlb uses an additional non-null struct sock pointer. */
+bool __skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio, struct sock *sk);
+static inline bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
+{
+	return __skb_page_frag_refill(sz, pfrag, prio, NULL);
+}
 
 /**
  * __skb_frag_dma_map - maps a paged fragment via the DMA API
include/linux/swiotlb.h +74 -0

100 lines not on this page · raw message

include/net/sock.h +29 -0
index dccd3738c3687..1e6caf4bd1366 100644
@@ -47,6 +47,7 @@
 #include <linux/skbuff.h>	/* struct sk_buff */
 #include <linux/mm.h>
 #include <linux/security.h>
+#include <linux/swiotlb.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/page_counter.h>
@@ -70,6 +71,14 @@
 #include <net/l3mdev.h>
 #include <uapi/linux/socket.h>
 
+#ifdef CONFIG_SWIOTLB
+struct sk_swiotlb_info {
+	struct device		*dev;
+	u32			serial;
+	unsigned long		jiffies;
+};
+#endif
+
 /*
  * This structure really needs to be cleaned up.
  * Most of it is for TCP, and not used by any of
@@ -602,8 +611,28 @@ struct sock {
 #if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES)
 	struct module		*sk_owner;
 #endif
+#ifdef CONFIG_SWIOTLB
+	struct sk_swiotlb_info	sk_swiotlb;
+#endif
 };
 
+#ifdef CONFIG_SWIOTLB
+static inline void sk_init_bounce_device(struct sock *sk)
+{
+	sk->sk_swiotlb.dev = NULL;
+}
+static inline void sk_cleanup_bounce_device(struct sock *sk)
+{
+	if (sk->sk_swiotlb.dev) {
+		swiotlb_safe_put_device(sk->sk_swiotlb.dev);
+		sk->sk_swiotlb.dev = NULL;
+	}
+}
+#else
+static inline void sk_init_bounce_device(struct sock *sk) {}
+static inline void sk_cleanup_bounce_device(struct sock *sk) {}
+#endif
+
 struct sock_bh_locked {
 	struct sock *sock;
 	local_lock_t bh_lock;
kernel/dma/swiotlb.c +227 -0

266 lines not on this page · raw message

mm/page_alloc.c +32 -0
index d49c254174da7..eaba683b5b2a8 100644
@@ -16,6 +16,7 @@
 
 #include <linux/stddef.h>
 #include <linux/mm.h>
+#include <linux/swiotlb.h>
 #include <linux/highmem.h>
 #include <linux/interrupt.h>
 #include <linux/jiffies.h>
@@ -705,6 +706,31 @@ void prep_compound_page(struct page *page, unsigned int order)
 	prep_compound_head(page, order);
 }
 
+#ifdef CONFIG_SWIOTLB
+void swiotlb_prep_compound_page(struct page *page, unsigned int order)
+{
+	if (order > 0)
+		prep_compound_page(page, order);
+}
+
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order)
+{
+	if (order > 0) {
+		struct folio *folio = (struct folio *)page;
+
+		__ClearPageHead(page);
+		page[1].flags.f &= ~PAGE_FLAGS_SECOND;
+#ifdef NR_PAGES_IN_LARGE_FOLIO
+		folio->_nr_pages = 0;
+#endif
+		for (int i = 1; i < (1 << order); i++) {
+			page[i].mapping = NULL;
+			clear_compound_head(&page[i]);
+		}
+	}
+}
+#endif /* CONFIG_SWIOTLB */
+
 static inline void set_buddy_order(struct page *page, unsigned int order)
 {
 	set_page_private(page, order);
@@ -2930,6 +2956,9 @@ static void __free_frozen_pages(struct page *page, unsigned int order,
 	unsigned long pfn = page_to_pfn(page);
 	int migratetype;
 
+	if (unlikely(swiotlb_free_pages(page, false)))
+		return;
+
 	if (!pcp_allowed_order(order)) {
 		__free_pages_ok(page, order, fpi_flags);
 		return;
@@ -2996,6 +3025,9 @@ void free_unref_folios(struct folio_batch *folios)
 		unsigned long pfn = folio_pfn(folio);
 		unsigned int order = folio_order(folio);
 
+		if (unlikely(swiotlb_free_pages(&folio->page, true)))
+			continue;
+
 		if (!__free_pages_prepare(&folio->page, order, FPI_NONE))
 			continue;
 		/*
net/core/sock.c +90 -8

148 lines not on this page · raw message

Signature · 1 line
2.54.0.1136.gdb2ca164c4-goog
maintainer
#1 lore raw
Quoted 4 lines · On Mon, 15 Jun 2026 23:42:20 +0000 Luigi Rizzo wrote:

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

What's the use case? I associate swiotlb with debug / testing mostly, so it'd be useful for people like me to explain why you care.

BTW net-next is closed: https://netdev.bots.linux.dev/net-next.html

#2 lore raw
Quoted 9 lines · Jakub Kicinski
Quoted 4 lines · On Mon, 15 Jun 2026 23:42:20 +0000 Luigi Rizzo wrote:

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

What's the use case? I associate swiotlb with debug / testing mostly, so it'd be useful for people like me to explain why you care.

Ah sorry, I forgot to mention.
swiotlb is used in guest kernels for confidential computing VMs. Ordinary memory pages are encrypted and the host or devices have no way to decrypt them, so the kernel must use unencrypted bounce buffers to exchange data with I/O devices.

cheers
luigi

#3 lore raw
Quoted 8 lines · Luigi Rizzo
Quoted 7 lines · Jakub Kicinski
Quoted 4 lines · On Mon, 15 Jun 2026 23:42:20 +0000 Luigi Rizzo wrote:

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Quoted 8 lines
Quoted 2 lines

What's the use case? I associate swiotlb with debug / testing mostly, so it'd be useful for people like me to explain why you care.

Ah sorry, I forgot to mention.
swiotlb is used in guest kernels for confidential computing VMs. Ordinary memory pages are encrypted and the host or devices have no way to decrypt them, so the kernel must use unencrypted bounce buffers to exchange data with I/O devices.

I started looking into the same problem recently, to reduce the bouncing in protected KVM (pKVM) confidential guests.
My first attempt was to update dma_direct_map_phys() to skip bouncing and do inline memory decryption (for pKVM that is a hypercall which updates the stage-2 page tables), however, that was really slow compared to the memcpy in bouncing even for massive pages. My conclusion was similar that we need to solve this at construction by making this memory allocated from a pre-decrypted pool (which does not have to be part of the SWIOTLB)
My initial idea was to teach some of the kernel subsystems (SKB, BLK, SLAB) about "CoCo allocators" that allocate decrypted memory, as this is not a net specific problem.

I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

Thanks,
Mostafa

Quoted 4 lines

cheers
luigi

#4 lore raw
Quoted 18 lines · Mostafa Saleh
Quoted 8 lines · Luigi Rizzo
Quoted 7 lines · Jakub Kicinski
Quoted 4 lines · On Mon, 15 Jun 2026 23:42:20 +0000 Luigi Rizzo wrote:

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Quoted 8 lines
Quoted 2 lines

What's the use case? I associate swiotlb with debug / testing mostly, so it'd be useful for people like me to explain why you care.

Ah sorry, I forgot to mention.
swiotlb is used in guest kernels for confidential computing VMs. Ordinary memory pages are encrypted and the host or devices have no way to decrypt them, so the kernel must use unencrypted bounce buffers to exchange data with I/O devices.

Quoted 13 lines

I started looking into the same problem recently, to reduce the bouncing in protected KVM (pKVM) confidential guests.
My first attempt was to update dma_direct_map_phys() to skip bouncing and do inline memory decryption (for pKVM that is a hypercall which updates the stage-2 page tables), however, that was really slow compared to the memcpy in bouncing even for massive pages. My conclusion was similar that we need to solve this at construction by making this memory allocated from a pre-decrypted pool (which does not have to be part of the SWIOTLB)
My initial idea was to teach some of the kernel subsystems (SKB, BLK, SLAB) about "CoCo allocators" that allocate decrypted memory, as this is not a net specific problem.

An example of this is Jiri's system_cc_shared heap which is a dma-buf heap with decrypted memory for userspace.

Quoted 5 lines

I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

There is a talk at LPC in the networking track about this [2]. This is exactly the type of discussion that I was hoping to have there.

Besides the issues mentioned in this thread we've also found that a lot of overhead can come only from swiotlb allocations when running many queues.

I will add information about this series in my talk. Hopefully I will also have time to add some numbers for comparison.

Sorry for the late reply but I spotted this thread only now by accident.

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lpc.events/event/20/contributions/2464

Thanks,
Dragos

#5 lore raw
Quoted 1 line · Dragos Tatulea

...

Quoted 14 lines

An example of this is Jiri's system_cc_shared heap which is a dma-buf heap with decrypted memory for userspace.

Quoted 5 lines

I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

There is a talk at LPC in the networking track about this [2]. This is exactly the type of discussion that I was hoping to have there.

Besides the issues mentioned in this thread we've also found that a lot of overhead can come only from swiotlb allocations when running many queues.

Quoted 8 lines

I will add information about this series in my talk. Hopefully I will also have time to add some numbers for comparison.

Sorry for the late reply but I spotted this thread only now by accident.

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lpc.events/event/20/contributions/2464

Thank you for the feedback and links.

Do you have some code describing [2] ?

I just sent a cleaned-up v2 of my previous series, if you want to try it please use the latter.

cheers
luigi

#6 lore raw
Quoted 23 lines · Luigi Rizzo
Quoted 1 line · Dragos Tatulea

...

Quoted 20 lines

An example of this is Jiri's system_cc_shared heap which is a dma-buf heap with decrypted memory for userspace.

Quoted 5 lines

I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

There is a talk at LPC in the networking track about this [2]. This is exactly the type of discussion that I was hoping to have there.

Besides the issues mentioned in this thread we've also found that a lot of overhead can come only from swiotlb allocations when running many queues.

I will add information about this series in my talk. Hopefully I will also have time to add some numbers for comparison.

Sorry for the late reply but I spotted this thread only now by accident.

Quoted 7 lines
Quoted 2 lines

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lpc.events/event/20/contributions/2464

Thank you for the feedback and links.

Do you have some code describing [2] ?

Not yet I will send a link once I push it somewhere. But the idea is simple:
a driver side alternative data-path which uses pre-allocated coherent DMA buffers. This doesn't make memcpy go away but eliminates the swiotlb all together and it reduces the DMA map/unmap overhead.

Quoted 3 lines

I just sent a cleaned-up v2 of my previous series, if you want to try it please use the latter.

Thanks! Had a quick lock and wrote some comments.

Thanks,
Dragos

#7 lore raw
In reply to Dragos Tatulea
Quoted 37 lines · Dragos Tatulea
Quoted 18 lines · Mostafa Saleh
Quoted 8 lines · Luigi Rizzo
Quoted 7 lines · Jakub Kicinski
Quoted 4 lines · On Mon, 15 Jun 2026 23:42:20 +0000 Luigi Rizzo wrote:

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Quoted 8 lines
Quoted 2 lines

What's the use case? I associate swiotlb with debug / testing mostly, so it'd be useful for people like me to explain why you care.

Ah sorry, I forgot to mention.
swiotlb is used in guest kernels for confidential computing VMs. Ordinary memory pages are encrypted and the host or devices have no way to decrypt them, so the kernel must use unencrypted bounce buffers to exchange data with I/O devices.

Quoted 13 lines

I started looking into the same problem recently, to reduce the bouncing in protected KVM (pKVM) confidential guests.
My first attempt was to update dma_direct_map_phys() to skip bouncing and do inline memory decryption (for pKVM that is a hypercall which updates the stage-2 page tables), however, that was really slow compared to the memcpy in bouncing even for massive pages. My conclusion was similar that we need to solve this at construction by making this memory allocated from a pre-decrypted pool (which does not have to be part of the SWIOTLB)
My initial idea was to teach some of the kernel subsystems (SKB, BLK, SLAB) about "CoCo allocators" that allocate decrypted memory, as this is not a net specific problem.

An example of this is Jiri's system_cc_shared heap which is a dma-buf heap with decrypted memory for userspace.

Quoted 7 lines
Quoted 5 lines

I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

There is a talk at LPC in the networking track about this [2]. This is exactly the type of discussion that I was hoping to have there.

I see, thanks for point that. I plan to be in LPC, so I will aim to attend this talk.

Quoted 6 lines

Besides the issues mentioned in this thread we've also found that a lot of overhead can come only from swiotlb allocations when running many queues.

I will add information about this series in my talk. Hopefully I will also have time to add some numbers for comparison.

I had a quick look, I am not sure how that will shape at the end, but I think we need a general solution beyond NICs. For example in pKVM the bouncing is used with virtio devices, so doing this per-driver won't really work and is not possible in some scenarios where memory is allocated from the core kernel and then passed to the driver. So, I was thinking the kernel relying on CC_ATTR_MEM_ENCRYPT/force_dma_unencrypted() could detect that and allocate pre-shared memory for those cases.

Thanks,
Mostafa

Quoted 9 lines

Sorry for the late reply but I spotted this thread only now by accident.

[1] https://lore.kernel.org/all/[email protected]/
[2] https://lpc.events/event/20/contributions/2464

Thanks,
Dragos

#8 lore raw
Quoted 11 lines · Mostafa Saleh
Quoted 10 lines · Dragos Tatulea
Quoted 7 lines · Mostafa Saleh

On Tue, Jun 16, 2026 at 02:33:52AM +0200, Luigi Rizzo wrote:
[...]
I am still looking into this, I was planning to bring this up in the upcoming LPC.
I will give this patch a try. However, I believe that we need a more generalised concept for CoCo pre-decrypted allocators in the kernel.

Quoted 6 lines
Quoted 2 lines

There is a talk at LPC in the networking track about this [2]. This is exactly the type of discussion that I was hoping to have there.

I see, thanks for point that. I plan to be in LPC, so I will aim to attend this talk.

Great!

Quoted 7 lines
Quoted 6 lines

Besides the issues mentioned in this thread we've also found that a lot of overhead can come only from swiotlb allocations when running many queues.

I will add information about this series in my talk. Hopefully I will also have time to add some numbers for comparison.

Quoted 9 lines

I had a quick look, I am not sure how that will shape at the end, but I think we need a general solution beyond NICs. For example in pKVM the bouncing is used with virtio devices, so doing this per-driver won't really work and is not possible in some scenarios where memory is allocated from the core kernel and then passed to the driver. So, I was thinking the kernel relying on CC_ATTR_MEM_ENCRYPT/force_dma_unencrypted() could detect that and allocate pre-shared memory for those cases.

Ideally it shouldn't be per driver. But I was thinking that there should be a way for the user to provision memory for certain devices.

Thanks,
Dragos

#9 lore raw
In reply to Luigi Rizzo
Quoted 10 lines · Luigi Rizzo

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent

Strange name, because your patch targets the regular tcp sendmsg() path (with a user -> kernel copy).

Typical high performance RPC libraries use TCP TX zerocopy these days. They won't benefit from this idea.
Perhaps you should state this in your changelog or documentation.

Also, what is the typical size of the bounce buffers in your guests?

With standard tcp_wmem settings, each TCP flow can consume 4 MB.

Quoted 9 lines

(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

Quoted 3 lines

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

This patch is too large; please split it into smaller functional units, so that each domain experts
can focus on their part.

I see you test SOCK_ZEROCOPY, but some applications setting this flag can mix tcp sendmsg() with or without zero-copy.

I also see your patch missed CONFIG_PREEMPT_RT case.

#10 lore raw
In reply to Luigi Rizzo
kernel test robot noticed the following build warnings:

Hi Luigi,

kernel test robot noticed the following build warnings:

[auto build test WARNING on akpm-mm/mm-everything]
[also build test WARNING on linus/master v7.1 next-20260615]
[cannot apply to driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Luigi-Rizzo/swiotlb-avoid-double-copy-with-swiotlb-on-tx-socket/20260616-074655
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260615234220.3946885-1-lrizzo%40google.com
patch subject: [PATCH] swiotlb: avoid double copy with swiotlb on tx socket
config: powerpc-pmac32_defconfig (https://download.01.org/0day-ci/archive/20260616/[email protected]/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project e19d1f51a2c80b63cd8ca95bcc757b7077112808)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260616/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> Warning: net/core/sock.c:3215 function parameter 'sk' not described in '__skb_page_frag_refill'
>> Warning: net/core/sock.c:3215 expecting prototype for skb_page_frag_refill(). Prototype was for __skb_page_frag_refill() instead

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

#11 lore raw
In reply to Luigi Rizzo
kernel test robot noticed the following build warnings:

Hi Luigi,

kernel test robot noticed the following build warnings:

[auto build test WARNING on akpm-mm/mm-everything]
[also build test WARNING on linus/master v7.1 next-20260615]
[cannot apply to driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Luigi-Rizzo/swiotlb-avoid-double-copy-with-swiotlb-on-tx-socket/20260616-074655
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260615234220.3946885-1-lrizzo%40google.com
patch subject: [PATCH] swiotlb: avoid double copy with swiotlb on tx socket
config: loongarch-allnoconfig (https://download.01.org/0day-ci/archive/20260616/[email protected]/config)
compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260616/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

>> mm/page_alloc.c:721:17: warning: unused variable 'folio' [-Wunused-variable]
     721 |                 struct folio *folio = (struct folio *)page;
         |                               ^~~~~
   1 warning generated.
--
>> Warning: kernel/dma/swiotlb.c:95 cannot understand function prototype: 'atomic_t global_device_serial = ATOMIC_INIT(0);'
>> Warning: kernel/dma/swiotlb.c:2087 function parameter 'where_debug_only' not described in 'swiotlb_free_pages'
>> Warning: kernel/dma/swiotlb.c:2087 function parameter 'where_debug_only' not described in 'swiotlb_free_pages'

vim +/folio +721 mm/page_alloc.c

717
718 void swiotlb_destroy_compound_page(struct page *page, unsigned int order)
719 {
720 if (order > 0) {

 > 721			struct folio *folio = (struct folio *)page;

722
723 __ClearPageHead(page);
724 page[1].flags.f &= ~PAGE_FLAGS_SECOND;
725 #ifdef NR_PAGES_IN_LARGE_FOLIO
726 folio->_nr_pages = 0;
727 #endif
728 for (int i = 1; i < (1 << order); i++) {
729 page[i].mapping = NULL;
730 clear_compound_head(&page[i]);
731 }
732 }
733 }
734 #endif /* CONFIG_SWIOTLB */
735

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

#12 lore raw
In reply to Luigi Rizzo
Quoted 580 lines · Luigi Rizzo

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent
(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

---
 drivers/base/core.c        |   1 +
 include/linux/netdevice.h  |  22 ++++
 include/linux/page-flags.h |   4 +
 include/linux/skbuff.h     |   7 +-
 include/linux/swiotlb.h    |  74 ++++++++++++
 include/net/sock.h         |  29 +++++
 kernel/dma/swiotlb.c       | 227 +++++++++++++++++++++++++++++++++++++
 mm/page_alloc.c            |  32 ++++++
 net/core/sock.c            |  98 ++++++++++++++--
 9 files changed, 485 insertions(+), 9 deletions(-)
drivers/base/core.c +1 -0
index bd2ddf2aab505..e1257dea37ba0 100644
@@ -3855,6 +3855,7 @@ void device_del(struct device *dev)
 	unsigned int noio_flag;
 
 	device_lock(dev);
+	swiotlb_device_deleted();
 	kill_device(dev);
 	device_unlock(dev);
 
include/linux/netdevice.h +22 -0
index 0e1e581efc5ac..d7e5929e73c92 100644
@@ -5368,13 +5368,35 @@ static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
 	return ops->ndo_start_xmit(skb, dev);
 }
 
+struct sock;
+
+#ifdef CONFIG_SWIOTLB
+/* Per-CPU pointer to the socket currently performing transmission.
+ * Used to bridge the networking and DMA layers, allowing the dma_map_page()
+ * path to identify the socket originating the packet and apply SWIOTLB optimizations.
+ */
+DECLARE_PER_CPU(struct sock *, current_tx_socket);
+static inline struct sock *__set_current_tx_socket(struct sock *sk)
+{
+	struct sock *old_sk = this_cpu_read(current_tx_socket);
+
+	this_cpu_write(current_tx_socket, sk);
+	return old_sk;
+}
+#else
+static inline struct sock *__set_current_tx_socket(struct sock *sk) { return NULL; }
+#endif
+
 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
 					    struct netdev_queue *txq, bool more)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
+	struct sock *old_sk;
 	netdev_tx_t rc;
 
+	old_sk = __set_current_tx_socket(skb->sk);
 	rc = __netdev_start_xmit(ops, skb, dev, more);
+	__set_current_tx_socket(old_sk);
 	if (rc == NETDEV_TX_OK)
 		txq_trans_update(dev, txq);
 
include/linux/page-flags.h +4 -0
index 7223f6f4e2b40..0ecbb404038a0 100644
@@ -923,6 +923,7 @@ enum pagetype {
 	PGTY_zsmalloc		= 0xf6,
 	PGTY_unaccepted		= 0xf7,
 	PGTY_large_kmalloc	= 0xf8,
+	PGTY_zcswiotlb		= 0xf9,
 
 	PGTY_mapcount_underflow = 0xff
 };
@@ -1055,6 +1056,9 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
 PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
 
+/* Pages in socket buffers from the swiotlb pool. */
+PAGE_TYPE_OPS(ZCSwiotlb, zcswiotlb, zcswiotlb)
+
 /**
  * PageHuge - Determine if the page belongs to hugetlbfs
  * @page: The page to test.
include/linux/skbuff.h +6 -1
index 3f06254ab1b72..62340909409e5 100644
@@ -3787,7 +3787,12 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto,
 	fragto->netmem = fragfrom->netmem;
 }
 
-bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
+/* zerocopy swiotlb uses an additional non-null struct sock pointer. */
+bool __skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio, struct sock *sk);
+static inline bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
+{
+	return __skb_page_frag_refill(sz, pfrag, prio, NULL);
+}
 
 /**
  * __skb_frag_dma_map - maps a paged fragment via the DMA API
include/linux/swiotlb.h +74 -0
index 3dae0f592063e..bd2d0e160a9d8 100644
@@ -7,8 +7,10 @@
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/limits.h>
+#include <linux/percpu.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
+#include <linux/atomic.h>
 
 struct device;
 struct page;
@@ -122,6 +124,9 @@ struct io_tlb_mem {
 	atomic_long_t total_used;
 	atomic_long_t used_hiwater;
 	atomic_long_t transient_nslabs;
+#else
+	unsigned long last_used_slots;
+	unsigned long last_used_jiffies;
 #endif
 };
 
@@ -185,6 +190,69 @@ bool is_swiotlb_active(struct device *dev);
 void __init swiotlb_adjust_size(unsigned long size);
 phys_addr_t default_swiotlb_base(void);
 phys_addr_t default_swiotlb_limit(void);
+
+/* Helpers for zerocopy swiotlb. */
+/* Control allocation fraction. */
+extern unsigned int swiotlb_zc_tx_percent;
+
+/* Track freshness of the leaf device info. */
+extern atomic_t global_device_serial;
+
+static inline u32 swiotlb_get_device_serial(void)
+{
+	return atomic_read(&global_device_serial);
+}
+
+static inline void swiotlb_device_deleted(void)
+{
+	atomic_inc(&global_device_serial);
+}
+
+struct page *swiotlb_alloc_pages(struct device *dev, unsigned int order);
+bool swiotlb_free_pages(struct page *page, bool where_debug_only);
+void swiotlb_safe_put_device(struct device *dev);
+
+static inline void swiotlb_set_page_dev(struct page *page, struct device *dev)
+{
+	page->private = (unsigned long)dev;
+}
+
+static inline struct device *swiotlb_page_to_dev(struct page *page)
+{
+	return (struct device *)compound_head(page)->private;
+}
+
+static inline bool is_zerocopy_swiotlb_folio(struct page *page)
+{
+	struct folio *folio = page_folio(page);
+
+	return folio_test_zcswiotlb(folio) && folio->private != 0;
+}
+
+/* These two are in mm/page_alloc.c */
+void swiotlb_prep_compound_page(struct page *page, unsigned int order);
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
+
+#if defined(CONFIG_NET)
+/*
+ * Track the socket for the currently transmitted packet, so the dma mapping
+ * function can record there the leaf device if it needs bounce buffers.
+ */
+struct sock;
+DECLARE_PER_CPU(struct sock *, current_tx_socket);
+void sk_set_bounce_device(struct sock *sk, struct device *dev);
+static inline void dma_learn_bounce_device(struct device *dev)
+{
+	struct sock *sk = this_cpu_read(current_tx_socket);
+
+	if (sk)
+		sk_set_bounce_device(sk, dev);
+}
+#else
+static inline void dma_learn_bounce_device(struct device *dev) {}
+#endif
+/* End helpers for zerocopy swiotlb. */
+
 #else
 static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
 {
@@ -234,6 +302,12 @@ static inline phys_addr_t default_swiotlb_limit(void)
 {
 	return 0;
 }
+
+/* zerocopy swiotlb stubs */
+static inline bool swiotlb_free_pages(struct page *page, int reason) { return false; }
+static inline u32 swiotlb_get_device_serial(void) { return 0; }
+static inline void swiotlb_device_deleted(void) {}
+
 #endif /* CONFIG_SWIOTLB */
 
 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
include/net/sock.h +29 -0
index dccd3738c3687..1e6caf4bd1366 100644
@@ -47,6 +47,7 @@
 #include <linux/skbuff.h>	/* struct sk_buff */
 #include <linux/mm.h>
 #include <linux/security.h>
+#include <linux/swiotlb.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/page_counter.h>
@@ -70,6 +71,14 @@
 #include <net/l3mdev.h>
 #include <uapi/linux/socket.h>
 
+#ifdef CONFIG_SWIOTLB
+struct sk_swiotlb_info {
+	struct device		*dev;
+	u32			serial;
+	unsigned long		jiffies;
+};
+#endif
+
 /*
  * This structure really needs to be cleaned up.
  * Most of it is for TCP, and not used by any of
@@ -602,8 +611,28 @@ struct sock {
 #if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES)
 	struct module		*sk_owner;
 #endif
+#ifdef CONFIG_SWIOTLB
+	struct sk_swiotlb_info	sk_swiotlb;
+#endif
 };
 
+#ifdef CONFIG_SWIOTLB
+static inline void sk_init_bounce_device(struct sock *sk)
+{
+	sk->sk_swiotlb.dev = NULL;
+}
+static inline void sk_cleanup_bounce_device(struct sock *sk)
+{
+	if (sk->sk_swiotlb.dev) {
+		swiotlb_safe_put_device(sk->sk_swiotlb.dev);
+		sk->sk_swiotlb.dev = NULL;
+	}
+}
+#else
+static inline void sk_init_bounce_device(struct sock *sk) {}
+static inline void sk_cleanup_bounce_device(struct sock *sk) {}
+#endif
+
 struct sock_bh_locked {
 	struct sock *sock;
 	local_lock_t bh_lock;
kernel/dma/swiotlb.c +227 -0
index 1abd3e6146f45..e27f23d03c482 100644
@@ -37,12 +37,16 @@
 #include <linux/mm.h>
 #include <linux/pfn.h>
 #include <linux/rculist.h>
+#include <linux/refcount.h>
 #include <linux/scatterlist.h>
 #include <linux/set_memory.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/swiotlb.h>
+#include <linux/moduleparam.h>
+#include <linux/percpu.h>
 #include <linux/types.h>
+#include <linux/atomic.h>
 #ifdef CONFIG_DMA_RESTRICTED_POOL
 #include <linux/of.h>
 #include <linux/of_fdt.h>
@@ -81,6 +85,17 @@ struct io_tlb_slot {
 static bool swiotlb_force_bounce;
 static bool swiotlb_force_disable;
 
+/**
+ * global_device_serial - Global sequence number for device deletions
+ *
+ * Incremented every time a device is unregistered (in device_del()).
+ * Used by subsystems (like SWIOTLB zero-copy sockets) as a fast, lockless
+ * O(1) cache invalidation serial to detect when a cached device pointer
+ * might have been deleted and needs to be expired to prevent Use-After-Free.
+ */
+atomic_t global_device_serial = ATOMIC_INIT(0);
+EXPORT_SYMBOL(global_device_serial);
+
 #ifdef CONFIG_SWIOTLB_DYNAMIC
 
 static void swiotlb_dyn_alloc(struct work_struct *work);
@@ -1442,6 +1457,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 	offset &= (IO_TLB_SIZE - 1);
 	index += pad_slots;
 	pool->slots[index].pad_slots = pad_slots;
+	/* Fix an upstream bug with alloc_align_mask = 0xffff */
+	pool->slots[index].alloc_size = mapping_size;
 	for (i = 0; i < (nr_slots(size) - pad_slots); i++)
 		pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
 	tlb_addr = slot_addr(pool->start, index) + offset;
@@ -1554,6 +1571,13 @@ void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
 		size_t mapping_size, enum dma_data_direction dir,
 		unsigned long attrs, struct io_tlb_pool *pool)
 {
+	/*
+	 * Recognize and avoid unmapping pages allocated for Zero-Copy SWIOTLB Page Bypass.
+	 * They will be eventually released when the page reference count drops to 0.
+	 */
+	if (is_zerocopy_swiotlb_folio(pfn_to_page(PHYS_PFN(tlb_addr))))
+		return;
+
 	/*
 	 * First, sync the memory before unmapping the entry
 	 */
@@ -1597,6 +1621,21 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
 	phys_addr_t swiotlb_addr;
 	dma_addr_t dma_addr;
 
+	dma_learn_bounce_device(dev);
+
+	/*
+	 * If the page was allocated via Zero-Copy SWIOTLB Page Bypass, it is likely
+	 * already good for DMA so we can return its dma address.
+	 */
+	if (is_zerocopy_swiotlb_folio(pfn_to_page(PHYS_PFN(paddr)))) {
+		dma_addr = phys_to_dma_unencrypted(dev, paddr);
+		if (likely(dma_capable(dev, dma_addr, size, true))) {
+			if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+				arch_sync_dma_for_device(paddr, size, dir);
+			return dma_addr;
+		}
+	}
+
 	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
 
 	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
@@ -1899,3 +1938,191 @@ static const struct reserved_mem_ops rmem_swiotlb_ops = {
 
 RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", &rmem_swiotlb_ops);
 #endif /* CONFIG_DMA_RESTRICTED_POOL */
+
+/*
+ * Asynchronous/Deferred Device Release.
+ * put_device() can trigger the final release path of a device which may sleep.
+ * Since SWIOTLB pages can be freed in atomic or interrupt context (e.g. TX completion),
+ * we must defer the put_device() call to task context using a workqueue.
+ */
+struct swiotlb_deferred_put {
+	struct work_struct work;
+	struct device *dev;
+};
+
+static void swiotlb_deferred_put_work(struct work_struct *work)
+{
+	struct swiotlb_deferred_put *dp = container_of(work, struct swiotlb_deferred_put, work);
+
+	put_device(dp->dev);
+	kfree(dp);
+}
+
+/**
+ * swiotlb_safe_put_device() - Safely release device reference from atomic/interrupt context
+ * @dev: The device structure to release.
+ *
+ * Enqueues a deferred put_device() call on a workqueue using GFP_ATOMIC.
+ * If memory allocation fails, the reference is leaked to avoid an immediate crash.
+ */
+void swiotlb_safe_put_device(struct device *dev)
+{
+	struct swiotlb_deferred_put *dp;
+
+	if (!dev)
+		return;
+
+	/*
+	 * FAST PATH (O(1) lockless): If this is not the last reference,
+	 * we can decrement it atomically and safely in any context
+	 * without allocating memory or scheduling work!
+	 */
+	if (refcount_dec_not_one(&dev->kobj.kref.refcount))
+		return;
+
+	/*
+	 * SLOW PATH: It is the last reference (refcount == 1). We must
+	 * defer the final put_device() to task context because it will
+	 * trigger device_release() which can sleep.
+	 */
+	dp = kmalloc_obj(*dp, GFP_ATOMIC);
+	if (dp) {
+		INIT_WORK(&dp->work, swiotlb_deferred_put_work);
+		dp->dev = dev;
+		schedule_work(&dp->work);
+	} else {
+		pr_warn_ratelimited("swiotlb: failed to allocate deferred put, leaking device ref\n");
+	}
+}
+EXPORT_SYMBOL_GPL(swiotlb_safe_put_device);
+
+unsigned int swiotlb_zc_tx_percent;
+module_param_named(zerocopy_tx_percent, swiotlb_zc_tx_percent, uint, 0644);
+
+static unsigned long fast_mem_used(struct io_tlb_mem *mem)
+{
+#ifdef CONFIG_DEBUG_FS
+	return mem_used(mem);
+#else
+	unsigned long last_j = READ_ONCE(mem->last_used_jiffies);
+	unsigned long now = jiffies;
+
+	if (time_after(now, last_j + HZ / 100) &&
+	    try_cmpxchg(&mem->last_used_jiffies, &last_j, now)) {
+		WRITE_ONCE(mem->last_used_slots, mem_used(mem));
+	}
+	return READ_ONCE(mem->last_used_slots);
+#endif
+}
+
+/**
+ * swiotlb_alloc_pages() - Allocate long-lived contiguous pages from SWIOTLB pool
+ * @dev: Device which requires the SWIOTLB bounce buffers.
+ * @order: Allocation order (log2 of number of pages).
+ */
+struct page *swiotlb_alloc_pages(struct device *dev, unsigned int order)
+{
+	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+	struct io_tlb_pool *pool;
+	int npages = 1 << order;
+	unsigned int max_pct;
+	phys_addr_t tlb_addr;
+	struct page *page;
+	int index;
+
+	if (!mem || !mem->nslabs)
+		return NULL;
+
+	max_pct = clamp(READ_ONCE(swiotlb_zc_tx_percent), 0u, 90u);
+	if (max_pct == 0 || max_pct * mem->nslabs <= fast_mem_used(mem) * 100)
+		return NULL;
+
+	/*
+	 * Enforce natural alignment for compound pages. The mask-based
+	 * compound_head() optimization (used when HVO is enabled and struct page
+	 * size is a power of 2) assumes that compound pages are naturally aligned
+	 * to their size. Without this, compound_head() on tail pages can return
+	 * a wrong head page pointer, leading to refcount corruption.
+	 */
+	index = swiotlb_find_slots(dev, 0, PAGE_SIZE * npages, ~(PAGE_MASK << order), &pool);
+	if (index == -1)
+		return NULL;
+
+	tlb_addr = slot_addr(pool->start, index);
+
+	pool->slots[index].pad_slots = 0;
+	pool->slots[index].alloc_size = PAGE_SIZE * npages;
+
+	page = pfn_to_page(PHYS_PFN(tlb_addr));
+
+	set_page_count(page, 1);
+
+	/* Strictly tag page[0] to prevent clobbering folio tail overlays */
+	__SetPageZCSwiotlb(page);
+
+	swiotlb_set_page_dev(page, dev);
+	get_device(dev);
+	swiotlb_prep_compound_page(page, order);
+	return page;
+}
+EXPORT_SYMBOL_GPL(swiotlb_alloc_pages);
+
+/*
+ * Debugging to track how swiotlb_free_pages() was called.
+ * b2: 0 from __free_frozen_pages(), 1 from free_unref_folios()
+ * b1: pool found b0: dev present,
+ */
+static unsigned long zc_debug[8];
+static int ctrs_num = 8;
+module_param_array(zc_debug, ulong, &ctrs_num, 0644);
+static void __zc_debug_stats(bool where, bool has_dev, bool has_pool)
+{
+	zc_debug[has_dev + has_pool * 2 + where * 4]++;
+}
+
+/**
+ * swiotlb_free_pages() - Free pages allocated via swiotlb_alloc_pages()
+ * @page: The starting struct page to release.
+ */
+bool swiotlb_free_pages(struct page *page, bool where_debug_only)
+{
+	struct page *head = compound_head(page);
+	struct device *dev = swiotlb_page_to_dev(head);
+	phys_addr_t head_tlb_addr = page_to_phys(head);
+	struct io_tlb_pool *pool;
+	int index, npages, i;
+
+	if (!folio_test_zcswiotlb(page_folio(head)))
+		return false;
+
+	pool = dev ? swiotlb_find_pool(dev, head_tlb_addr) : NULL;
+	__zc_debug_stats(where_debug_only, !!dev, !!pool);
+
+	/* Check for any false positives. */
+	if (!pool)
+		return false;
+
+	/* Read alloc_size first, it is reset by swiotlb_release_slots(). */
+	index = (head_tlb_addr - pool->start) >> IO_TLB_SHIFT;
+	npages = pool->slots[index].alloc_size >> PAGE_SHIFT;
+
+	WARN_ON_ONCE(!is_power_of_2(npages));
+
+	/* Step 1: Sever compound links (clobbers compound_info / lru.next) */
+	swiotlb_destroy_compound_page(head, ilog2(npages));
+
+	/* Step 2: Re-init LRU, drop refcounts, and strip flag across all constituent pages */
+	for (i = 0; i < npages; i++) {
+		INIT_LIST_HEAD(&head[i].lru);
+		set_page_count(&head[i], 0);
+		head[i].private = 0;
+		__ClearPageZCSwiotlb(&head[i]);
+	}
+
+	/* Step 3: Safely release slots back to the pool */
+	swiotlb_release_slots(dev, head_tlb_addr, pool);
+	swiotlb_del_transient(dev, head_tlb_addr, pool);
+	swiotlb_safe_put_device(dev);
+	return true;
+}
+EXPORT_SYMBOL_GPL(swiotlb_free_pages);
mm/page_alloc.c +1 -0
index d49c254174da7..eaba683b5b2a8 100644
@@ -16,6 +16,7 @@
 
 #include <linux/stddef.h>
 #include <linux/mm.h>
+#include <linux/swiotlb.h>
 #include <linux/highmem.h>
 #include <linux/interrupt.h>
 #include <linux/jiffies.h>

mm/page_alloc.c void prep_compound_page(struct page *page, unsigned int order) · line 714

Quoted 9 lines
mm/page_alloc.c +6 -0
@@ -705,6 +706,31 @@ void prep_compound_page(struct page *page, unsigned int order)
 	prep_compound_head(page, order);
 }
 
+#ifdef CONFIG_SWIOTLB
+void swiotlb_prep_compound_page(struct page *page, unsigned int order)
+{
+	if (order > 0)
+		prep_compound_page(page, order);
+}

Gah.

Quoted 7 lines
mm/page_alloc.c +7 -0
+
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order)
+{
+	if (order > 0) {
+		struct folio *folio = (struct folio *)page;
+
+		__ClearPageHead(page);

mm/page_alloc.c void prep_compound_page(struct page *page, unsigned int order) · line 731

Quoted 10 lines
mm/page_alloc.c +10 -0
+		page[1].flags.f &= ~PAGE_FLAGS_SECOND;
+#ifdef NR_PAGES_IN_LARGE_FOLIO
+		folio->_nr_pages = 0;
+#endif
+		for (int i = 1; i < (1 << order); i++) {
+			page[i].mapping = NULL;
+			clear_compound_head(&page[i]);
+		}
+	}
+}

Gah.

mm/page_alloc.c static void __free_frozen_pages(struct page *page, unsigned int order, · line 2961

Quoted 12 lines
mm/page_alloc.c +5 -0
+#endif /* CONFIG_SWIOTLB */
+
 static inline void set_buddy_order(struct page *page, unsigned int order)
 {
 	set_page_private(page, order);
@@ -2930,6 +2956,9 @@ static void __free_frozen_pages(struct page *page, unsigned int order,
 	unsigned long pfn = page_to_pfn(page);
 	int migratetype;
 
+	if (unlikely(swiotlb_free_pages(page, false)))
+		return;
+

Oh my.

We shouldn't be handling randomg swiotlb stuff in the page allocator like that.

IIUC, you are writing your own pool+allocator and roughly mimic what hugetlb + ZONE_DEVICE does.

The creation+destruction of compound pages should very likely be factored out from other code in a type-unspecific fashion, if really required.

You should probably look into

https://lore.kernel.org/all/[email protected]/

to see how to possibly hook into the page freeing path in a cleaner way.

Signature · 3 lines
Cheers,

David
#13 lore raw
In reply to Luigi Rizzo

(+cc page pool maintainers)

Quoted 18 lines · Luigi Rizzo

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent
(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

Quoted 3 lines

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

I could be wrong, but I genuinely think that the way to go about this is using page_pool for regular TX as well. page_pool pages are all dma-mapped (so whatever swiotlb optimization you want can be done there), and the net stack already has awareness of these special pages and special skbs, so it won't Just Return Them back to the page allocator.

Otherwise you can easily go all over the place, and that's just not great. Also this could possibly benefit setups that use IOMMU as well.

Signature · 1 line
Pedro
#14 lore raw
Quoted 25 lines · Pedro Falcato

(+cc page pool maintainers)

Quoted 18 lines · Luigi Rizzo

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent
(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

Quoted 3 lines

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

Quoted 5 lines

I could be wrong, but I genuinely think that the way to go about this is using page_pool for regular TX as well. page_pool pages are all dma-mapped (so whatever swiotlb optimization you want can be done there), and the net stack already has awareness of these special pages and special skbs, so it won't Just Return Them back to the page allocator.

I am not sure I follow your comment above, can you expand/clarify?

The problem I am dealing with is that the copy from the socket buffer to the bounce buffer is done in the device xmit function. Under high it is almost always done by the tx softirq.
This means that even if we move the copy outside the HARD_TX_LOCK(), it would still be almost completely serialized.
Hence the proposed method to make skb_page_frag_refill() allocate directly a bounce buffer (under specific conditions) so there is a single copy done directly to the dma-able buffer, and ii is done in the user threads/CPUs and is not seriallized in the softirq thread.

I am not sure how page_pool on tx could help here.

cheers
luigi

#15 lore raw
Quoted 44 lines · Luigi Rizzo
Quoted 25 lines · Pedro Falcato

(+cc page pool maintainers)

Quoted 18 lines · Luigi Rizzo

The use of swiotlb causes an extra data copy on I/O. For tx sockets, especially with greedy senders, this has a high chance of happening in the softirq handler for tx network interrupts, creating a significant performance bottleneck.

Allow tx sockets to allocate socket buffers directly from the bounce buffers. This avoids the second copy and removes the above bottleneck. The fraction of swiotlb buffers allowed for this feature is set with
/sys/module/swiotlb/parameters/zerocopy_tx_percent
(0 means disabled, 90 is the maximum, to avoid persistent I/O failures).

Implementation:
- define a new page type to unambiguously identify bounce buffers used as backing storage for socket buffers
- modify skb_page_frag_refill to perform the modified allocation
- modify the destructors __free_frozen_pages(), free_unref_folio() to handle those pages and return them to the pool.

Quoted 3 lines

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

Quoted 5 lines

I could be wrong, but I genuinely think that the way to go about this is using page_pool for regular TX as well. page_pool pages are all dma-mapped (so whatever swiotlb optimization you want can be done there), and the net stack already has awareness of these special pages and special skbs, so it won't Just Return Them back to the page allocator.

I am not sure I follow your comment above, can you expand/clarify?

The problem I am dealing with is that the copy from the socket buffer to the bounce buffer is done in the device xmit function. Under high it is almost always done by the tx softirq.
This means that even if we move the copy outside the HARD_TX_LOCK(), it would still be almost completely serialized.
Hence the proposed method to make skb_page_frag_refill() allocate directly a bounce buffer (under specific conditions) so there is a single copy done directly to the dma-able buffer, and ii is done in the user threads/CPUs and is not seriallized in the softirq thread.

Quoted 1 line

I am not sure how page_pool on tx could help here.

Page pool would provide both the means of passing around an iommu-mapped page, and a concrete "this is where we allocate these pages" spot. Then introducing a "zero-copy" swiotlb allocation would be a simple matter of introducing this on page pool's side. In pseudo-code, something like:

static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
gfp_t gfp)

{
	struct page *page;

	gfp |= __GFP_COMP;
	
	if (pool->dma_map && /* is_swiotlb */) {
		page = swiotlb_alloc_pages(pool->p.nid, gfp, pool->p.order, ...);
		if (!page)
			return NULL;
		/* page is implicitly swiotlb mapped (well, _actually_ it's
		 * not that simple, because of the dma_mapped tracking that
		 * was introduced, but PoC anyway..). */
	} else {
		page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
		if (unlikely(!page))
			return NULL;

		if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page), gfp))) {
			put_page(page);
			return NULL;
		}
	}
}

(plus other spots, obviously). No copying should be required, and the netmem desc will keep the dma_addr around. The network stack will notice pp_recycle on all of these skbs and simply refuse to throw the pages away to the page allocator.

In any case, it might be that this is not feasible for XYZ reasons, but I've thought about this (making net use and reuse page pool pre-iommu-mapped pages exclusively) for a while and I definitely see a lot of similarities with your problem (that more or less reduces down to "I want to get an iommu-mapped page from the get-go").

Signature · 1 line
Pedro
#16 lore raw
In reply to Luigi Rizzo
kernel test robot noticed the following build warnings:

Hi Luigi,

kernel test robot noticed the following build warnings:

[auto build test WARNING on akpm-mm/mm-everything]
[also build test WARNING on linus/master v7.1 next-20260615]
[cannot apply to driver-core/driver-core-testing driver-core/driver-core-next driver-core/driver-core-linus] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Luigi-Rizzo/swiotlb-avoid-double-copy-with-swiotlb-on-tx-socket/20260616-074655
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260615234220.3946885-1-lrizzo%40google.com
patch subject: [PATCH] swiotlb: avoid double copy with swiotlb on tx socket
config: arm-randconfig-r122-20260616 (https://download.01.org/0day-ci/archive/20260616/[email protected]/config)
compiler: arm-linux-gnueabi-gcc (GCC) 16.1.0
sparse: v0.6.5-rc1
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260616/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

sparse warnings: (new ones prefixed by >>)
kernel/dma/swiotlb.c: note: in included file (through include/linux/dma-direct.h):

>> include/linux/swiotlb.h:229:65: sparse: sparse: Using plain integer as NULL pointer
>> include/linux/swiotlb.h:229:65: sparse: sparse: Using plain integer as NULL pointer

vim +229 include/linux/swiotlb.h

224
225 static inline bool is_zerocopy_swiotlb_folio(struct page *page)
226 {
227 struct folio *folio = page_folio(page);
228

 > 229		return folio_test_zcswiotlb(folio) && folio->private != 0;

230 }
231

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

#17 lore raw

v2 cover swiotlb: avoid swiotlb copy on network sockets

In reply to Luigi Rizzo

The use of swiotlb, common in Confidential Computing, causes an extra data copy on each I/O. Focusing on network sockets:
- on tx, the copy has a high chance of happening in the tx softirq handler (especially with greedy senders where the device queue is often full)
- on rx, it is guaranteed to happen in the rx softirq handler. Thus, on top of the copy cost, swiotlb concentrates the overhead on an already constrained resource (CPUs processing network interrupts).

Reduce or remove the extra copy by conditionally allocating socket buffers directly from the swiotlb buffer pool.

The feature is controlled by runtime parameters to set the percentage of swiotlb buffers that can be used for this purpose. This avoids stranding the entire swiotlb pool in socket buffers.

The implementation is made of four main parts:
- introduce a swiotlb page allocator that can be used instead of regular pages, and teach __free_frozen_pages(), free_unref_folio() how to handle them
- dynamically track the leaf device for each tx network socket, so we can tell at copy_from_user() time whether we need to use swiotlb for this socket
- modify skb_page_frag_refill() to allocate from swiotlb if needed. This implements the copy elision for the transmit path
- modify __page_pool_alloc_page_order() to allocate from swiotlb if needed. This implements the copy elision for the receive path.

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

OPEN ISSUES

Currently the swiotlb allocator looks for free slots using an approximately linear scan of each pool (with some hints to likely candidates) and then does a linear scan of subsequent pools. This works extremely well when the number of pools matches the number of CPUs, and there is plenty of memory available. In fact, it is almost unbeatable by any more complex strategy.

Under high load or buffer fragmentation, a CPU might repeatedly do a full scan of its starting pool before finding a suitable candidate. Even worse, with multiple tx/rx queues, what happens is that multiple CPUs will trail each other on the same sequence of pools. The effect is that some allocations will end up costing O(100us) and more.

I have tried to implement two improvements:
- a buddy allocator on top of each pool, so to make it quicker to find a candidate of the requested size
- make each CPU use a different sequence to explore other pools in case one is full, so they will not end up queueing one after the other
While they are very effective on the tails, for low load scenarios the current linear allocators is better. Thus this will take more investigation.

---
v1 -> v2:

- split components into separate commits
- simplified allocator, no need for a new page type
- many code cleanups
- also implement the rx side

Luigi Rizzo (5):
swiotlb: enforce pool nareas and nslabs invariants swiotlb/mm: Implement SWIOTLB nocopy page allocator net/swiotlb: Track bounce device per socket
net: Divert socket allocations to SWIOTLB for nocopy TX
swiotlb: Implement RX nocopy with fast recycling eviction

 drivers/base/core.c       |   1 +
 drivers/iommu/dma-iommu.c |   9 +-
 include/linux/netdevice.h |  21 +++
 include/linux/skbuff.h    |   7 +-
 include/linux/swiotlb.h   |  63 ++++++++
 include/net/sock.h        |  46 ++++++
 kernel/dma/direct.h       |  11 ++
 kernel/dma/swiotlb.c      | 296 ++++++++++++++++++++++++++++++++++++--
 mm/page_alloc.c           |  61 +++++++-
 net/core/page_pool.c      |  25 +++-
 net/core/sock.c           | 101 +++++++++++--
 11 files changed, 617 insertions(+), 24 deletions(-)
Signature · 1 line
2.55.0.766.g2966f0265a-goog
#18 lore raw

PATCH v2 1/5 swiotlb: enforce pool nareas and nslabs invariants

The SWIOTLB allocator relies on two runtime invariants across all pool initialization paths:

1. pool->nareas must always be a power of two so that a slot's area can be located efficiently via bitwise masking (index & (nareas - 1)) instead of integer division.
2. pool->nslabs must be a multiple of nareas * IO_TLB_SEGSIZE so that each area contains an integer multiple of IO_TLB_SEGSIZE (default 128) slots, preventing contiguous allocations from crossing area boundaries.

Enforce these invariants consistently during early boot, pool initialization (swiotlb_init_io_tlb_pool), and restricted DMA pool setup.

Fixes 8ac04063354a ("swiotlb: reduce the number of areas to match actual memory pool size")
---
 kernel/dma/swiotlb.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)
kernel/dma/swiotlb.c +16 -3
index 1abd3e6146f45..8e4bd9d47735a 100644
@@ -33,6 +33,7 @@
 #include <linux/kmsan-checks.h>
 #include <linux/iommu-helper.h>
 #include <linux/init.h>
+#include <linux/log2.h>
 #include <linux/memblock.h>
 #include <linux/mm.h>
 #include <linux/pfn.h>
@@ -176,7 +177,7 @@ static void swiotlb_adjust_nareas(unsigned int nareas)
 static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
 {
 	if (nslots < nareas * IO_TLB_SEGSIZE)
-		return nslots / IO_TLB_SEGSIZE;
+		return rounddown_pow_of_two(nslots / IO_TLB_SEGSIZE);
 	return nareas;
 }
 
@@ -269,7 +270,16 @@ static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
 		unsigned long nslabs, bool late_alloc, unsigned int nareas)
 {
 	void *vaddr = phys_to_virt(start);
-	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
+	unsigned long bytes, i;
+
+	/*
+	 * If we have multiple areas, ensure each area's size is a multiple of
+	 * IO_TLB_SEGSIZE slots by aligning the total pool size down.
+	 */
+	if (nareas > 1)
+		nslabs = ALIGN_DOWN(nslabs, nareas * IO_TLB_SEGSIZE);
+
+	bytes = nslabs << IO_TLB_SHIFT;
 
 	mem->nslabs = nslabs;
 	mem->start = start;
@@ -1813,7 +1823,10 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
 				    struct device *dev)
 {
 	struct io_tlb_mem *mem = rmem->priv;
-	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
+	unsigned long nslabs = round_down(rmem->size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
+
+	if (!nslabs)
+		return -EINVAL;
 
 	/* Set Per-device io tlb area to one */
 	unsigned int nareas = 1;
Signature · 1 line
2.55.0.766.g2966f0265a-goog
#19 lore raw

PATCH v2 2/5 swiotlb/mm: Implement SWIOTLB nocopy page allocator

In reply to Luigi Rizzo

Introduce swiotlb_alloc_pages() and swiotlb_free_pages() to allocate and release compound pages directly from the default SWIOTLB pool.

The allocator is restricted to slots in the static default pool, with caller-specified percentage limits on pool occupancy.

This will be used for kernel data (e.g. socket buffers) in nocopy confidential computing.

---
 include/linux/swiotlb.h |  24 ++++++
 kernel/dma/swiotlb.c    | 187 ++++++++++++++++++++++++++++++++++++++--
 mm/page_alloc.c         |  51 +++++++++++
 3 files changed, 255 insertions(+), 7 deletions(-)
include/linux/swiotlb.h +24 -0
index 3dae0f592063e..4661e361c60e4 100644
@@ -169,6 +169,23 @@ static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
 	return NULL;
 }
 
+bool swiotlb_pool_is_nocopy(struct io_tlb_pool *pool, phys_addr_t paddr);
+
+static inline bool swiotlb_addr_in_default_pool(struct device *dev,
+						phys_addr_t paddr)
+{
+	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+
+	return mem && paddr >= mem->defpool.start && paddr < mem->defpool.end;
+}
+
+static inline bool swiotlb_is_nocopy_addr(struct device *dev, phys_addr_t paddr)
+{
+	if (!swiotlb_addr_in_default_pool(dev, paddr))
+		return false;
+	return swiotlb_pool_is_nocopy(&dev->dma_io_tlb_mem->defpool, paddr);
+}
+
 static inline bool is_swiotlb_force_bounce(struct device *dev)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
@@ -178,6 +195,13 @@ static inline bool is_swiotlb_force_bounce(struct device *dev)
 
 void swiotlb_init(bool addressing_limited, unsigned int flags);
 void __init swiotlb_exit(void);
+struct page *swiotlb_alloc_pages(struct device *dev, unsigned int order, gfp_t gfp,
+				 unsigned int percent);
+bool swiotlb_free_pages(struct page *page, unsigned int order);
+void swiotlb_nocopy_inc_ref(struct io_tlb_pool *pool, phys_addr_t phys);
+void swiotlb_nocopy_dec_ref(struct io_tlb_pool *pool, phys_addr_t phys);
+void swiotlb_prep_compound_page(struct page *page, unsigned int order);
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
 void swiotlb_dev_init(struct device *dev);
 size_t swiotlb_max_mapping_size(struct device *dev);
 bool is_swiotlb_allocated(void);
kernel/dma/swiotlb.c +180 -7

286 lines not on this page · raw message

mm/page_alloc.c +51 -0

63 lines not on this page · raw message

Signature · 1 line
2.55.0.766.g2966f0265a-goog
maintainer
#20 lore raw

Re: [PATCH v2 2/5] swiotlb/mm: Implement SWIOTLB nocopy page allocator

Quoted 2 lines · Luigi Rizzo

Introduce swiotlb_alloc_pages() and swiotlb_free_pages() to allocate and release compound pages directly from the default SWIOTLB pool.

Huh? The sole intended purpose of SWIOTLB is for bounce-buffering data which already exists in some other memory that is unsuitable for DMA for whatever reason. If you want to allocate directly from some kind of pre-shared DMA page pool to avoid bounce-buffering, set up some kind of pre-shared DMA page pool and allocate from that in a manner which can avoid bouncing entirely (see DMA_ATTR_CC_SHARED). The idea of getting as far as calling swiotlb_bounce() to then have a special case saying "haha not really" seems entirely absurd. Don't hack stuff into the SWIOTLB code which has no business being there.

Thanks,
Robin.

Quoted 428 lines

The allocator is restricted to slots in the static default pool, with caller-specified percentage limits on pool occupancy.

This will be used for kernel data (e.g. socket buffers) in nocopy confidential computing.

---
  include/linux/swiotlb.h |  24 ++++++
  kernel/dma/swiotlb.c    | 187 ++++++++++++++++++++++++++++++++++++++--
  mm/page_alloc.c         |  51 +++++++++++
  3 files changed, 255 insertions(+), 7 deletions(-)
include/linux/swiotlb.h +24 -0
index 3dae0f592063e..4661e361c60e4 100644
@@ -169,6 +169,23 @@ static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
  	return NULL;
  }
 
+bool swiotlb_pool_is_nocopy(struct io_tlb_pool *pool, phys_addr_t paddr);
+
+static inline bool swiotlb_addr_in_default_pool(struct device *dev,
+						phys_addr_t paddr)
+{
+	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+
+	return mem && paddr >= mem->defpool.start && paddr < mem->defpool.end;
+}
+
+static inline bool swiotlb_is_nocopy_addr(struct device *dev, phys_addr_t paddr)
+{
+	if (!swiotlb_addr_in_default_pool(dev, paddr))
+		return false;
+	return swiotlb_pool_is_nocopy(&dev->dma_io_tlb_mem->defpool, paddr);
+}
+
  static inline bool is_swiotlb_force_bounce(struct device *dev)
  {
  	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
@@ -178,6 +195,13 @@ static inline bool is_swiotlb_force_bounce(struct device *dev)
 
  void swiotlb_init(bool addressing_limited, unsigned int flags);
  void __init swiotlb_exit(void);
+struct page *swiotlb_alloc_pages(struct device *dev, unsigned int order, gfp_t gfp,
+				 unsigned int percent);
+bool swiotlb_free_pages(struct page *page, unsigned int order);
+void swiotlb_nocopy_inc_ref(struct io_tlb_pool *pool, phys_addr_t phys);
+void swiotlb_nocopy_dec_ref(struct io_tlb_pool *pool, phys_addr_t phys);
+void swiotlb_prep_compound_page(struct page *page, unsigned int order);
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
  void swiotlb_dev_init(struct device *dev);
  size_t swiotlb_max_mapping_size(struct device *dev);
  bool is_swiotlb_allocated(void);
kernel/dma/swiotlb.c +180 -7
index 8e4bd9d47735a..6b86a1e955fb4 100644
@@ -66,17 +66,59 @@
  /**
   * struct io_tlb_slot - IO TLB slot descriptor
   * @orig_addr:	The original address corresponding to a mapped entry.
+ * @nocopy_refcnt:	Lockless atomic refcount for Nocopy buffers.
   * @alloc_size:	Size of the allocated buffer.
   * @list:	The free list describing the number of free entries available
   *		from each index.
   * @pad_slots:	Number of preceding padding slots. Valid only in the first
   *		allocated non-padding slot.
+ * @flags:	Slot attributes (e.g. SWIOTLB_SLOT_NOCOPY for Nocopy buffers).
+ *
+ * The slot descriptor has states identified by @list and @flags (SWIOTLB_SLOT_NOCOPY):
+ *
+ * 1. FREE (list > 0):
+ *    Linear sweep free slot.
+ *
+ * 2. USED (list == 0, SWIOTLB_SLOT_NOCOPY flag is NOT set in @flags):
+ *    Allocated SWIOTLB bounce buffer.
+ *    Fields used: @list, @pad_slots, @orig_addr, @alloc_size.
+ *
+ * 3. USED_NOCOPY (list == 0, SWIOTLB_SLOT_NOCOPY flag is set in @flags):
+ *    Allocated Nocopy SWIOTLB buffer.
+ *    Fields used: @list, @nocopy_refcnt, @alloc_size.
+ */
+#define SWIOTLB_SLOT_NOCOPY	BIT(0)
+
+/*
+ * SWIOTLB nocopy allocations (swiotlb_alloc_pages()) do not have an original
+ * physical address to bounce, but need to pass a caller-specified pool usage
+ * limit (percentage) down to the area search logic.
+ *
+ * To avoid adding a parameter to swiotlb_find_slots(), swiotlb_search_area(),
+ * and swiotlb_search_pool_area(), the desired percentage (0..90) is encoded
+ * into the orig_addr parameter in the reserved high address range starting at
+ * INVALID_PHYS_ADDR (~0ULL).
+ *
+ * - NOCOPY_PCT_TO_ADDR(pct): Encodes a percentage into an orig_addr.
+ * - IS_SWIOTLB_NOCOPY(addr): Identifies a nocopy allocation request and
+ *   restricts slot search to the static default pool.
+ * - NOCOPY_ADDR_TO_PCT(addr): Extracts the percentage to cap max_usable
+ *   slots in swiotlb_search_pool_area().
   */
+#define NOCOPY_PCT_MAX	(90u)
+#define NOCOPY_PCT_TO_ADDR(pct)		(INVALID_PHYS_ADDR - min(pct, NOCOPY_PCT_MAX))
+#define IS_SWIOTLB_NOCOPY(addr)		((addr) >= INVALID_PHYS_ADDR - NOCOPY_PCT_MAX)
+#define NOCOPY_ADDR_TO_PCT(addr)	((unsigned int)(INVALID_PHYS_ADDR - (addr)))
+
  struct io_tlb_slot {
-	phys_addr_t orig_addr;
+	union {
+		phys_addr_t orig_addr;
+		atomic_t nocopy_refcnt;
+	};
  	size_t alloc_size;
  	unsigned short list;
  	unsigned short pad_slots;
+	unsigned int flags;
  };
 
  static bool swiotlb_force_bounce;
@@ -300,6 +342,7 @@ static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
  		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
  		mem->slots[i].alloc_size = 0;
  		mem->slots[i].pad_slots = 0;
+		mem->slots[i].flags = 0;
  	}
 
  	memset(vaddr, 0, bytes);
@@ -869,12 +912,17 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
  			   enum dma_data_direction dir, struct io_tlb_pool *mem)
  {
  	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
-	phys_addr_t orig_addr = mem->slots[index].orig_addr;
  	size_t alloc_size = mem->slots[index].alloc_size;
-	unsigned long pfn = PFN_DOWN(orig_addr);
  	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
+	phys_addr_t orig_addr;
+	unsigned long pfn;
  	int tlb_offset;
 
+	/* Nocopy swiotlb buffers do not need bouncing. */
+	if (mem->slots[index].flags & SWIOTLB_SLOT_NOCOPY)
+		return;
+
+	orig_addr = mem->slots[index].orig_addr;
  	if (orig_addr == INVALID_PHYS_ADDR)
  		return;
 
@@ -904,6 +952,7 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
  		size = alloc_size;
  	}
 
+	pfn = PFN_DOWN(orig_addr);
  	if (PageHighMem(pfn_to_page(pfn))) {
  		unsigned int offset = orig_addr & ~PAGE_MASK;
  		struct page *page;
@@ -1052,7 +1101,8 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
  	unsigned long max_slots = get_max_slots(boundary_mask);
  	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
  	unsigned int nslots = nr_slots(alloc_size), stride;
-	unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
+	unsigned long max_usable = pool->area_nslabs;
+	unsigned int offset;
  	unsigned int index, slots_checked, count = 0, i;
  	unsigned long flags;
  	unsigned int slot_base;
@@ -1061,6 +1111,13 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
  	BUG_ON(!nslots);
  	BUG_ON(area_index >= pool->nareas);
 
+	if (IS_SWIOTLB_NOCOPY(orig_addr)) {
+		max_usable = (pool->area_nslabs * NOCOPY_ADDR_TO_PCT(orig_addr)) / 100;
+		orig_addr = 0;
+	}
+
+	offset = swiotlb_align_offset(dev, 0, orig_addr);
+
  	/*
  	 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
  	 * page-aligned in the absence of any other alignment requirements.
@@ -1087,7 +1144,7 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
  	stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
 
  	spin_lock_irqsave(&area->lock, flags);
-	if (unlikely(nslots > pool->area_nslabs - area->used))
+	if (unlikely(area->used + nslots > max_usable))
  		goto not_found;
 
  	slot_base = area_index * pool->area_nslabs;
@@ -1164,6 +1221,9 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
   * Search one memory area in all pools for a sequence of slots that match the
   * allocation constraints.
   *
+ * If IS_SWIOTLB_NOCOPY(orig_addr) is true, the search is restricted to only the
+ * default pool, which is what swiotlb_alloc_pages() is allowed to use.
+ *
   * Return: Index of the first allocated slot, or -1 on error.
   */
  static int swiotlb_search_area(struct device *dev, int start_cpu,
@@ -1177,6 +1237,9 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
 
  	rcu_read_lock();
  	list_for_each_entry_rcu(pool, &mem->pools, node) {
+		/* Only search the default pool (first in mem->pools) for nocopy allocations. */
+		if (IS_SWIOTLB_NOCOPY(orig_addr) && pool != &mem->defpool)
+			break;
  		if (cpu_offset >= pool->nareas)
  			continue;
  		area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
@@ -1229,6 +1292,13 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
  			goto found;
  	}
 
+	/*
+	 * Passing a nocopy orig_addr restricts the search to only the
+	 * default pool, so do not attempt dynamic pool expansion.
+	 */
+	if (IS_SWIOTLB_NOCOPY(orig_addr))
+		return -1;
+
  	if (!mem->can_grow)
  		return -1;
 
@@ -1468,11 +1538,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
  	return tlb_addr;
  }
 
+/*
+ * called with dev == NULL from swiotlb_dealloc_pages(), in this case force offset
+ * and align_mask to 0, pad_slots is also 0, and assume the pages come from the
+ * default system pool.
+ */
  static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
  				  struct io_tlb_pool *mem)
  {
  	unsigned long flags;
-	unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
+	unsigned int offset = dev ? swiotlb_align_offset(dev, 0, tlb_addr) : 0;
  	int index, nslots, aindex;
  	struct io_tlb_area *area;
  	int count, i;
@@ -1506,6 +1581,7 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
  		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
  		mem->slots[i].alloc_size = 0;
  		mem->slots[i].pad_slots = 0;
+		mem->slots[i].flags = 0;
  	}
 
  	/*
@@ -1519,7 +1595,7 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
  	area->used -= nslots;
  	spin_unlock_irqrestore(&area->lock, flags);
 
-	dec_used(dev->dma_io_tlb_mem, nslots);
+	dec_used(dev ? dev->dma_io_tlb_mem : &io_tlb_default_mem, nslots);
  }
 
  #ifdef CONFIG_SWIOTLB_DYNAMIC
@@ -1912,3 +1988,100 @@ static const struct reserved_mem_ops rmem_swiotlb_ops = {
 
  RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", &rmem_swiotlb_ops);
  #endif /* CONFIG_DMA_RESTRICTED_POOL */
+
+static inline int swiotlb_nocopy_head_index(struct io_tlb_pool *pool, phys_addr_t phys)
+{
+	return (page_to_phys(compound_head(phys_to_page(phys))) - pool->start) >> IO_TLB_SHIFT;
+}
+
+/**
+ * swiotlb_dealloc_pages() - Actually release Nocopy slots and page metadata
+ * @pool:	SWIOTLB pool containing the buffer.
+ * @parent:	Slot index of the buffer head.
+ */
+static void swiotlb_dealloc_pages(struct io_tlb_pool *pool, unsigned int parent)
+{
+	unsigned int order = get_order(pool->slots[parent].alloc_size);
+	phys_addr_t paddr = pool->start + (parent << IO_TLB_SHIFT);
+	struct page *head = phys_to_page(paddr);
+
+	swiotlb_destroy_compound_page(head, order);
+	swiotlb_release_slots(NULL, paddr, pool);
+}
+
+struct page *swiotlb_alloc_pages(struct device *dev, unsigned int order,
+				 gfp_t gfp, unsigned int percent)
+{
+	struct io_tlb_pool *pool;
+	struct page *page;
+	int index, nslots, i;
+
+	if (WARN_ON_ONCE(!dev || !dev->dma_io_tlb_mem))
+		return NULL;
+
+	if (dev->dma_io_tlb_mem != &io_tlb_default_mem)
+		return NULL;
+
+	index = swiotlb_find_slots(dev, NOCOPY_PCT_TO_ADDR(percent),
+				   PAGE_SIZE << order, (PAGE_SIZE << order) - 1,
+				   &pool);
+	if (index < 0)
+		return NULL;
+
+	nslots = (PAGE_SIZE << order) >> IO_TLB_SHIFT;
+	page = phys_to_page(pool->start + (index << IO_TLB_SHIFT));
+	swiotlb_prep_compound_page(page, order);
+	for (i = 0; i < nslots; i++)
+		pool->slots[index + i].flags |= SWIOTLB_SLOT_NOCOPY;
+	atomic_set(&pool->slots[index].nocopy_refcnt, 1);
+	return page;
+}
+EXPORT_SYMBOL(swiotlb_alloc_pages);
+
+bool swiotlb_free_pages(struct page *page, unsigned int order)
+{
+	struct io_tlb_mem *mem = &io_tlb_default_mem;
+	struct io_tlb_pool *pool = &mem->defpool;
+	struct page *head = compound_head(page);
+	unsigned int parent;
+	phys_addr_t paddr;
+
+	paddr = page_to_phys(head);
+	if (paddr < pool->start || paddr >= pool->end)
+		return false;
+
+	parent = swiotlb_nocopy_head_index(pool, paddr);
+	if (!(pool->slots[parent].flags & SWIOTLB_SLOT_NOCOPY))
+		return false;
+
+	if (atomic_dec_and_test(&pool->slots[parent].nocopy_refcnt))
+		swiotlb_dealloc_pages(pool, parent);
+
+	return true;
+}
+EXPORT_SYMBOL(swiotlb_free_pages);
+
+void swiotlb_nocopy_inc_ref(struct io_tlb_pool *pool, phys_addr_t phys)
+{
+	int head_idx = swiotlb_nocopy_head_index(pool, phys);
+
+	atomic_inc(&pool->slots[head_idx].nocopy_refcnt);
+}
+EXPORT_SYMBOL(swiotlb_nocopy_inc_ref);
+
+void swiotlb_nocopy_dec_ref(struct io_tlb_pool *pool, phys_addr_t phys)
+{
+	int head_idx = swiotlb_nocopy_head_index(pool, phys);
+
+	if (atomic_dec_and_test(&pool->slots[head_idx].nocopy_refcnt))
+		swiotlb_dealloc_pages(pool, head_idx);
+}
+EXPORT_SYMBOL(swiotlb_nocopy_dec_ref);
+
+bool swiotlb_pool_is_nocopy(struct io_tlb_pool *pool, phys_addr_t paddr)
+{
+	int index = (paddr - pool->start) >> IO_TLB_SHIFT;
+
+	return pool->slots[index].flags & SWIOTLB_SLOT_NOCOPY;
+}
+EXPORT_SYMBOL_GPL(swiotlb_pool_is_nocopy);
mm/page_alloc.c +51 -0
index 083cbcb5bddec..ea148562a76d0 100644
@@ -16,6 +16,7 @@
 
  #include <linux/stddef.h>
  #include <linux/mm.h>
+#include <linux/swiotlb.h>
  #include <linux/highmem.h>
  #include <linux/interrupt.h>
  #include <linux/jiffies.h>
@@ -711,6 +712,56 @@ void prep_compound_page(struct page *page, unsigned int order)
  	prep_compound_head(page, order);
  }
 
+#ifdef CONFIG_SWIOTLB
+/*
+ * Prepare a SWIOTLB page (potentially compound).
+ *
+ * We explicitly initialize the head page refcount to 1 because recycled
+ * SWIOTLB pages might have a refcount of 0.
+ *
+ * If order > 0 (compound page), we must explicitly set all tail page
+ * refcounts to 0. This is because SWIOTLB pages might have a boot-default
+ * refcount of 1, but the core memory management subsystem expects tail pages
+ * of a compound page to have a refcount of 0.
+ */
+void swiotlb_prep_compound_page(struct page *page, unsigned int order)
+{
+	init_page_count(page);
+	if (order > 0) {
+		for (int i = 1; i < (1 << order); i++)
+			set_page_count(page + i, 0);
+		prep_compound_page(page, order);
+	}
+}
+
+/*
+ * Destroy a SWIOTLB compound page and restore page refcounts.
+ *
+ * When pages are returned to the SWIOTLB pool, we restore the refcount of
+ * all constituent pages (head and tails) to 1. This resets them to their
+ * clean boot-default state, ensuring they are ready for reuse either as
+ * individual order-0 pages or as part of a new compound allocation.
+ */
+void swiotlb_destroy_compound_page(struct page *page, unsigned int order)
+{
+	if (order > 0) {
+		struct folio *folio = (struct folio *)page;
+
+		__ClearPageHead(page);
+		page[1].flags.f &= ~PAGE_FLAGS_SECOND;
+#ifdef NR_PAGES_IN_LARGE_FOLIO
+		folio->_nr_pages = 0;
+#endif
+		for (int i = 1; i < (1 << order); i++) {
+			page[i].mapping = NULL;
+			clear_compound_head(&page[i]);
+			set_page_count(page + i, 1);
+		}
+	}
+	set_page_count(page, 1);
+}
+#endif /* CONFIG_SWIOTLB */
+
  static inline void set_buddy_order(struct page *page, unsigned int order)
  {
  	set_page_private(page, order);
#21 lore raw

Re: [PATCH v2 2/5] swiotlb/mm: Implement SWIOTLB nocopy page allocator

Quoted 5 lines · Robin Murphy
Quoted 2 lines · Luigi Rizzo

Introduce swiotlb_alloc_pages() and swiotlb_free_pages() to allocate and release compound pages directly from the default SWIOTLB pool.

Quoted 9 lines

Huh? The sole intended purpose of SWIOTLB is for bounce-buffering data which already exists in some other memory that is unsuitable for DMA for whatever reason. If you want to allocate directly from some kind of pre-shared DMA page pool to avoid bounce-buffering, set up some kind of pre-shared DMA page pool and allocate from that in a manner which can avoid bouncing entirely (see DMA_ATTR_CC_SHARED). The idea of getting as far as calling swiotlb_bounce() to then have a special case saying "haha not really" seems entirely absurd. Don't hack stuff into the SWIOTLB code which has no business being there.

Ah I see DMA_ATTR_CC_SHARED did not exist when I implemented the swiotlb allocator. Cool, one less piece, it should be possible to replace this chunk witth the DMA_ATTR_CC_SHARED.

The other pieces are still relevant though ?

- use these dma-able pages for page pool allocations (small)
- [PATCH v2 3/5] net/swiotlb: Track bounce device per socket or one would have to unconditionally allocate from that pool, even for e.g. local sockets
- divert socket allocations for eligible sockets to a DMA-able pool

cheers
Luigi

#22 lore raw

Re: [PATCH v2 2/5] swiotlb/mm: Implement SWIOTLB nocopy page allocator

Quoted 16 lines · Luigi Rizzo
Quoted 5 lines · Robin Murphy
Quoted 2 lines · Luigi Rizzo

Introduce swiotlb_alloc_pages() and swiotlb_free_pages() to allocate and release compound pages directly from the default SWIOTLB pool.

Quoted 9 lines

Huh? The sole intended purpose of SWIOTLB is for bounce-buffering data which already exists in some other memory that is unsuitable for DMA for whatever reason. If you want to allocate directly from some kind of pre-shared DMA page pool to avoid bounce-buffering, set up some kind of pre-shared DMA page pool and allocate from that in a manner which can avoid bouncing entirely (see DMA_ATTR_CC_SHARED). The idea of getting as far as calling swiotlb_bounce() to then have a special case saying "haha not really" seems entirely absurd. Don't hack stuff into the SWIOTLB code which has no business being there.

Quoted 4 lines

Ah I see DMA_ATTR_CC_SHARED did not exist when I implemented the swiotlb allocator. Cool, one less piece, it should be possible to replace this chunk witth the DMA_ATTR_CC_SHARED.

I agree with Robin's point (if I understood it correctly):
Why should it at all pass through the SWIOTLB? Why can't it be a standalone DMA pool?

Quoted 8 lines

The other pieces are still relevant though ?

- use these dma-able pages for page pool allocations (small)
- [PATCH v2 3/5] net/swiotlb: Track bounce device per socket or one would have to unconditionally allocate from that pool, even for e.g. local sockets
- divert socket allocations for eligible sockets to a DMA-able pool

Thanks,
Dragos

#23 lore raw

PATCH v2 3/5 net/swiotlb: Track bounce device per socket

In reply to Luigi Rizzo

Record, on each network socket, the underlying hardware device that requests DMA mapping of tx packets. This is used for nocopy confidential computing, so that sockets can eventually allocate socket buffers directly from the swiotlb pools.

---
 drivers/base/core.c       |  1 +
 include/linux/netdevice.h | 21 +++++++++++
 include/linux/swiotlb.h   | 36 +++++++++++++++++++
 include/net/sock.h        | 46 ++++++++++++++++++++++++
 kernel/dma/swiotlb.c      | 73 +++++++++++++++++++++++++++++++++++++++
 net/core/sock.c           | 39 +++++++++++++++++++++
 6 files changed, 216 insertions(+)
drivers/base/core.c +1 -0
index 4c0c373998a19..091062228740d 100644
@@ -3925,6 +3925,7 @@ void device_del(struct device *dev)
 	unsigned int noio_flag;
 
 	device_lock(dev);
+	swiotlb_change_epoch();
 	kill_device(dev);
 	device_unlock(dev);
 
include/linux/netdevice.h +21 -0
index 87cafc932e9e6..2457f4e464acf 100644
@@ -5429,13 +5429,34 @@ static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
 	return ops->ndo_start_xmit(skb, dev);
 }
 
+struct sock;
+
+#if defined(CONFIG_SWIOTLB) && !defined(CONFIG_PREEMPT_RT)
+/* Per-CPU pointer to the socket currently performing transmission. Used
+ * to bridge the networking and DMA layers, allowing dma_map_page() to
+ * identify the socket originating the packet and apply SWIOTLB optimizations.
+ */
+DECLARE_PER_CPU(struct sock *, current_tx_socket);
+static inline struct sock *__save_current_tx_socket(struct sock *sk)
+{
+	struct sock *old_sk = this_cpu_read(current_tx_socket);
+
+	this_cpu_write(current_tx_socket, sk);
+	return old_sk;
+}
+#else
+static inline struct sock *__save_current_tx_socket(struct sock *sk) { return NULL; }
+#endif
+
 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
 					    struct netdev_queue *txq, bool more)
 {
+	struct sock *old_sk = __save_current_tx_socket(skb->sk);
 	const struct net_device_ops *ops = dev->netdev_ops;
 	netdev_tx_t rc;
 
 	rc = __netdev_start_xmit(ops, skb, dev, more);
+	__save_current_tx_socket(old_sk);
 	if (rc == NETDEV_TX_OK)
 		txq_trans_update(dev, txq);
 
include/linux/swiotlb.h +36 -0
index 4661e361c60e4..b1140db3cc397 100644
@@ -202,6 +202,39 @@ void swiotlb_nocopy_inc_ref(struct io_tlb_pool *pool, phys_addr_t phys);
 void swiotlb_nocopy_dec_ref(struct io_tlb_pool *pool, phys_addr_t phys);
 void swiotlb_prep_compound_page(struct page *page, unsigned int order);
 void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
+void swiotlb_safe_put_device(struct device *dev);
+
+/* Track epoch (number of delete operations) for leaf device info. */
+extern atomic_t global_device_epoch;
+
+static inline u32 swiotlb_dev_epoch(void)
+{
+	return atomic_read(&global_device_epoch);
+}
+
+static inline void swiotlb_change_epoch(void)
+{
+	atomic_inc(&global_device_epoch);
+}
+
+#if defined(CONFIG_NET) && !defined(CONFIG_PREEMPT_RT)
+/*
+ * Track the socket for the currently transmitted packet, so the dma mapping
+ * function can record there the leaf device if it needs bounce buffers.
+ */
+struct sock;
+DECLARE_PER_CPU(struct sock *, current_tx_socket);
+void sk_record_bounce_device(struct sock *sk, struct device *dev);
+static inline void dma_learn_bounce_device(struct device *dev)
+{
+	struct sock *sk = this_cpu_read(current_tx_socket);
+
+	if (sk)
+		sk_record_bounce_device(sk, dev);
+}
+#else
+static inline void dma_learn_bounce_device(struct device *dev) {}
+#endif
 void swiotlb_dev_init(struct device *dev);
 size_t swiotlb_max_mapping_size(struct device *dev);
 bool is_swiotlb_allocated(void);
@@ -258,6 +291,9 @@ static inline phys_addr_t default_swiotlb_limit(void)
 {
 	return 0;
 }
+static inline void swiotlb_safe_put_device(struct device *dev)
+{
+}
 #endif /* CONFIG_SWIOTLB */
 
 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
include/net/sock.h +46 -0

66 lines not on this page · raw message

kernel/dma/swiotlb.c +73 -0

82 lines not on this page · raw message

net/core/sock.c +39 -0
index 1ad41904db25b..ca3e08d3de141 100644
@@ -103,6 +103,8 @@
 #include <linux/sockios.h>
 #include <linux/net.h>
 #include <linux/mm.h>
+#include <linux/swiotlb.h>
+#include <linux/device.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/poll.h>
@@ -152,6 +154,41 @@
 
 #include "dev.h"
 
+#if defined(CONFIG_SWIOTLB) && !defined(CONFIG_PREEMPT_RT)
+
+DEFINE_PER_CPU(struct sock *, current_tx_socket);
+EXPORT_PER_CPU_SYMBOL(current_tx_socket);
+
+void sk_record_bounce_device(struct sock *sk, struct device *dev)
+{
+	struct device *old_dev;
+
+	if (in_hardirq() || !sk_fullsock(sk) || sock_flag(sk, SOCK_ZEROCOPY))
+		return;
+
+	old_dev = rcu_dereference_protected(sk->sk_swiotlb.dev, 1);
+
+	if (dev != old_dev) {
+		/* Rate-limit updates to once per second to prevent bonding thrashing */
+		if (old_dev && time_before(jiffies, sk->sk_swiotlb.jiffies + HZ))
+			return;
+
+		get_device(dev);
+
+		/* Atomically swap in the new device and get the actual old one */
+		old_dev = (struct device *)xchg((struct device __force **)&sk->sk_swiotlb.dev,
+						(struct device __force *)dev);
+
+		WRITE_ONCE(sk->sk_swiotlb.epoch, swiotlb_dev_epoch());
+		sk->sk_swiotlb.jiffies = jiffies;
+
+		/* Only drop the reference to the device we actually replaced */
+		if (old_dev)
+			swiotlb_safe_put_device(old_dev);
+	}
+}
+EXPORT_SYMBOL(sk_record_bounce_device);
+#endif
 static DEFINE_MUTEX(proto_list_mutex);
 static LIST_HEAD(proto_list);
 
@@ -2387,6 +2424,7 @@ static void __sk_destruct(struct rcu_head *head)
 		__netns_tracker_free(net, &sk->ns_tracker, false);
 		net_passive_dec(net);
 	}
+	sk_release_bounce_device(sk);
 	sk_prot_free(sk->sk_prot_creator, sk);
 }
 
@@ -2489,6 +2527,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority,
 		goto out;
 
 	sock_copy(newsk, sk);
+	sk_clear_bounce_device(newsk);
 
 	newsk->sk_prot_creator = prot;
 #ifdef CONFIG_BPF_SYSCALL
Signature · 1 line
2.55.0.766.g2966f0265a-goog
#24 lore raw

PATCH v2 4/5 net: Divert socket allocations to SWIOTLB for nocopy TX

In reply to Luigi Rizzo

Conditionally intercept socket buffer page allocations and direct them to the SWIOTLB page allocator when nocopy tx is active.

This only happens when the swiotlb usage is below module parameter swiotlb.nocopy_tx_percent (default 0, range 0..90)
A value of 0 disables the feature.

---
 drivers/iommu/dma-iommu.c |  9 +++++-
 include/linux/skbuff.h    |  7 ++++-
 include/linux/swiotlb.h   |  2 ++
 kernel/dma/direct.h       | 11 +++++++
 kernel/dma/swiotlb.c      | 12 ++++++++
 mm/page_alloc.c           | 10 ++++++-
 net/core/sock.c           | 62 ++++++++++++++++++++++++++++++++++-----
 7 files changed, 102 insertions(+), 11 deletions(-)
drivers/iommu/dma-iommu.c +8 -1
index 9a07eb39336eb..956d5e11b2896 100644
@@ -1228,7 +1228,14 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	 * If both the physical buffer start address and size are page aligned,
 	 * we don't need to use a bounce page.
 	 */
-	if (dev_use_swiotlb(dev, size, dir) &&
+	bool is_nocopy = false;
+
+	if (swiotlb_is_nocopy_addr(dev, phys)) {
+		swiotlb_nocopy_inc_ref(&dev->dma_io_tlb_mem->defpool, phys);
+		is_nocopy = true;
+	}
+
+	if (!is_nocopy && dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
 		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
 			return DMA_MAPPING_ERROR;
include/linux/skbuff.h +6 -1
index add0d282dea6e..d8f7041edc400 100644
@@ -3786,7 +3786,12 @@ static inline void skb_frag_page_copy(skb_frag_t *fragto,
 	fragto->netmem = fragfrom->netmem;
 }
 
-bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
+/* nocopy swiotlb uses an additional non-null struct sock pointer. */
+bool __skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio, struct sock *sk);
+static inline bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio)
+{
+	return __skb_page_frag_refill(sz, pfrag, prio, NULL);
+}
 
 /**
  * __skb_frag_dma_map - maps a paged fragment via the DMA API
include/linux/swiotlb.h +2 -0
index b1140db3cc397..3baf52e6572d0 100644
@@ -204,6 +204,8 @@ void swiotlb_prep_compound_page(struct page *page, unsigned int order);
 void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
 void swiotlb_safe_put_device(struct device *dev);
 
+extern unsigned int nocopy_tx_percent;
+
 /* Track epoch (number of delete operations) for leaf device info. */
 extern atomic_t global_device_epoch;
 
kernel/dma/direct.h +11 -0
index 7140c208c1238..21c65acb13823 100644
@@ -88,6 +88,17 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
 {
 	dma_addr_t dma_addr;
 
+	if (swiotlb_is_nocopy_addr(dev, phys)) {
+		dma_addr_t unenc_addr = phys_to_dma_unencrypted(dev, phys);
+
+		if (likely(dma_capable(dev, unenc_addr, size, true))) {
+			swiotlb_nocopy_inc_ref(&dev->dma_io_tlb_mem->defpool, phys);
+			if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+				arch_sync_dma_for_device(phys, size, dir);
+			return unenc_addr;
+		}
+	}
+
 	if (is_swiotlb_force_bounce(dev)) {
 		if (!(attrs & DMA_ATTR_CC_SHARED)) {
 			if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
kernel/dma/swiotlb.c +12 -0
index d4a07a7c570e1..7b818a796ff96 100644
@@ -63,6 +63,11 @@
  */
 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
 
+/* enable nocopy tx swiotlb and set the percentage of buffers allowed for it. */
+unsigned int nocopy_tx_percent;
+module_param(nocopy_tx_percent, uint, 0644);
+MODULE_PARM_DESC(nocopy_tx_percent, "percentage of swiotlb buffer allowed for nocopy tx");
+
 /**
  * struct io_tlb_slot - IO TLB slot descriptor
  * @orig_addr:	The original address corresponding to a mapped entry.
@@ -1640,6 +1645,13 @@ void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
 		size_t mapping_size, enum dma_data_direction dir,
 		unsigned long attrs, struct io_tlb_pool *pool)
 {
+	int index = (tlb_addr - pool->start) >> IO_TLB_SHIFT;
+
+	if (pool->slots[index].flags & SWIOTLB_SLOT_NOCOPY) {
+		swiotlb_nocopy_dec_ref(pool, tlb_addr);
+		return;
+	}
+
 	/*
 	 * First, sync the memory before unmapping the entry
 	 */
mm/page_alloc.c +9 -1
index ea148562a76d0..32d5d630f9840 100644
@@ -3002,9 +3002,14 @@ static void __free_frozen_pages(struct page *page, unsigned int order,
 {
 	struct per_cpu_pages *pcp;
 	struct zone *zone;
-	unsigned long pfn = page_to_pfn(page);
+	unsigned long pfn;
 	int migratetype;
 
+	if (unlikely(swiotlb_free_pages(page, order)))
+		return;
+
+	pfn = page_to_pfn(page);
+
 	if (!pcp_allowed_order(order)) {
 		__free_pages_ok(page, order, fpi_flags);
 		return;
@@ -3070,6 +3075,9 @@ void free_unref_folios(struct folio_batch *folios)
 		unsigned long pfn = folio_pfn(folio);
 		unsigned int order = folio_order(folio);
 
+		if (unlikely(swiotlb_free_pages(&folio->page, order)))
+			continue;
+
 		if (!__free_pages_prepare(&folio->page, order, FPI_NONE))
 			continue;
 		/*
net/core/sock.c +54 -8

94 lines not on this page · raw message

Signature · 1 line
2.55.0.766.g2966f0265a-goog
#25 lore raw

Re: [PATCH v2 4/5] net: Divert socket allocations to SWIOTLB for nocopy TX

kernel/dma/swiotlb.c line 69

Quoted 12 lines · Luigi Rizzo
kernel/dma/swiotlb.c +4 -0
index d4a07a7c570e1..7b818a796ff96 100644
@@ -63,6 +63,11 @@
  */
 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
 
+/* enable nocopy tx swiotlb and set the percentage of buffers allowed for it. */
+unsigned int nocopy_tx_percent;
+module_param(nocopy_tx_percent, uint, 0644);
+MODULE_PARM_DESC(nocopy_tx_percent, "percentage of swiotlb buffer allowed for nocopy tx");

Please add this module param to the existing swiotlb param documentation in Documentation/admin-guide/kernel-parameters.txt.

Signature · 1 line
~Randy
#26 lore raw

PATCH v2 5/5 swiotlb: Implement RX nocopy with fast recycling eviction

In reply to Luigi Rizzo

Conditionally divert receive buffer allocations in page_pool to the SWIOTLB page allocator.

This only happens when swiotlb usage is below the threshold set by module parameter swiotlb.nocopy_rx_percent (default 0, range 0..90). A value of 0 disables the feature.

To prevent existing DRAM or SWIOTLB pages from circulating indefinitely in the lockless receive ring after changing the parameter at runtime, __page_pool_put_page() checks residency against the active parameter state. Mismatched pages are immediately evicted back to their respective allocators, achieving rapid, lockless mode conversion across active network streams without requiring interface or queue resets.

---
 include/linux/swiotlb.h |  1 +
 kernel/dma/swiotlb.c    |  5 +++++
 net/core/page_pool.c    | 25 ++++++++++++++++++++++---
 3 files changed, 28 insertions(+), 3 deletions(-)
include/linux/swiotlb.h +1 -0
index 3baf52e6572d0..f4597fd01c52d 100644
@@ -205,6 +205,7 @@ void swiotlb_destroy_compound_page(struct page *page, unsigned int order);
 void swiotlb_safe_put_device(struct device *dev);
 
 extern unsigned int nocopy_tx_percent;
+extern unsigned int nocopy_rx_percent;
 
 /* Track epoch (number of delete operations) for leaf device info. */
 extern atomic_t global_device_epoch;
kernel/dma/swiotlb.c +5 -0
index 7b818a796ff96..91f175c34a34e 100644
@@ -129,6 +129,11 @@ struct io_tlb_slot {
 static bool swiotlb_force_bounce;
 static bool swiotlb_force_disable;
 
+/* enable nocopy rx swiotlb and set the percentage of buffers allowed for it. */
+unsigned int nocopy_rx_percent;
+module_param(nocopy_rx_percent, uint, 0644);
+MODULE_PARM_DESC(nocopy_rx_percent, "percentage of swiotlb buffer allowed for nocopy rx");
+
 #ifdef CONFIG_SWIOTLB_DYNAMIC
 
 static void swiotlb_dyn_alloc(struct work_struct *work);
net/core/page_pool.c +22 -3
index 50ee550fef73a..fe8839a7c70a8 100644
@@ -19,6 +19,7 @@
 
 #include <linux/dma-direction.h>
 #include <linux/dma-mapping.h>
+#include <linux/swiotlb.h>
 #include <linux/page-flags.h>
 #include <linux/mm.h> /* for put_page() */
 #include <linux/poison.h>
@@ -578,10 +579,16 @@ static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t g
 static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
 						 gfp_t gfp)
 {
+	unsigned int pct = READ_ONCE(nocopy_rx_percent);
 	struct page *page;
 
 	gfp |= __GFP_COMP;
-	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
+	page = NULL;
+	if (pct && is_swiotlb_active(pool->p.dev))
+		page = swiotlb_alloc_pages(pool->p.dev, pool->p.order, gfp,
+					   pct);
+	if (!page)
+		page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
 	if (unlikely(!page))
 		return NULL;
 
@@ -616,8 +623,9 @@ static noinline netmem_ref __page_pool_alloc_netmems_slow(struct page_pool *pool
 	if ((gfp & GFP_ATOMIC) == GFP_ATOMIC)
 		gfp |= __GFP_NOWARN;
 
-	/* Don't support bulk alloc for high-order pages */
-	if (unlikely(pp_order))
+	/* Don't support bulk alloc for high-order pages or nocopy SWIOTLB */
+	if (unlikely(pp_order || (READ_ONCE(nocopy_rx_percent) &&
+				  is_swiotlb_active(pool->p.dev))))
 		return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
 
 	/* Unnecessary as alloc cache is empty, but guarantees zero count */
@@ -835,6 +843,17 @@ __page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
 {
 	lockdep_assert_no_hardirq();
 
+	/*
+	 * If runtime nocopy mode toggled, evict circulating buffers immediately
+	 * back to their respective allocators rather than recycling them.
+	 */
+	if (unlikely(!netmem_is_net_iov(netmem) &&
+		     swiotlb_is_nocopy_addr(pool->p.dev, page_to_phys(netmem_to_page(netmem))) !=
+		     (READ_ONCE(nocopy_rx_percent) > 0))) {
+		page_pool_return_netmem(pool, netmem);
+		return 0;
+	}
+
 	/* This allocator is optimized for the XDP mode that uses
 	 * one-frame-per-page, but have fallbacks that act like the
 	 * regular page allocator APIs.
Signature · 1 line
2.55.0.766.g2966f0265a-goog
#27 lore raw

Re: [PATCH v2 5/5] swiotlb: Implement RX nocopy with fast recycling eviction

Quoted 7 lines · Luigi Rizzo

Conditionally divert receive buffer allocations in page_pool to the SWIOTLB page allocator.

This only happens when swiotlb usage is below the threshold set by module parameter swiotlb.nocopy_rx_percent (default 0, range 0..90). A value of 0 disables the feature.

Quoted 7 lines

To prevent existing DRAM or SWIOTLB pages from circulating indefinitely in the lockless receive ring after changing the parameter at runtime, __page_pool_put_page() checks residency against the active parameter state. Mismatched pages are immediately evicted back to their respective allocators, achieving rapid, lockless mode conversion across active network streams without requiring interface or queue resets.

This could be a separate page_pool allocator. It could be similar to how ZC works: pre-allocate some coherent DMA buffers and allocate from there.

Thanks,
Dragps

#28 lore raw

Re: [PATCH v2 0/5] swiotlb: avoid swiotlb copy on network sockets

In reply to Luigi Rizzo
Quoted 11 lines · Luigi Rizzo

The use of swiotlb, common in Confidential Computing, causes an extra data copy on each I/O. Focusing on network sockets:
- on tx, the copy has a high chance of happening in the tx softirq handler (especially with greedy senders where the device queue is often full)
- on rx, it is guaranteed to happen in the rx softirq handler. Thus, on top of the copy cost, swiotlb concentrates the overhead on an already constrained resource (CPUs processing network interrupts).

Reduce or remove the extra copy by conditionally allocating socket buffers directly from the swiotlb buffer pool.

Isn't it dangerous for RX to expose kernel structures to the HV? If SKB the linear area is exposed to the HW, the headroom and tailroom are up for grabs for the HV: the HV could modify them in a TOCTOU fashioon.

Quoted 29 lines

The feature is controlled by runtime parameters to set the percentage of swiotlb buffers that can be used for this purpose. This avoids stranding the entire swiotlb pool in socket buffers.

The implementation is made of four main parts:
- introduce a swiotlb page allocator that can be used instead of regular pages, and teach __free_frozen_pages(), free_unref_folio() how to handle them
- dynamically track the leaf device for each tx network socket, so we can tell at copy_from_user() time whether we need to use swiotlb for this socket
- modify skb_page_frag_refill() to allocate from swiotlb if needed. This implements the copy elision for the transmit path
- modify __page_pool_alloc_page_order() to allocate from swiotlb if needed. This implements the copy elision for the receive path.

The savings are especially visible with fewer queues. In synthetic benchmarks, senders with 1-2 queues would cap around 50Gbps with conventional swiotlb, and reach over 170Gbps with the feature enabled.

OPEN ISSUES

Currently the swiotlb allocator looks for free slots using an approximately linear scan of each pool (with some hints to likely candidates) and then does a linear scan of subsequent pools. This works extremely well when the number of pools matches the number of CPUs, and there is plenty of memory available. In fact, it is almost unbeatable by any more complex strategy.

Quoted 5 lines

Under high load or buffer fragmentation, a CPU might repeatedly do a full scan of its starting pool before finding a suitable candidate. Even worse, with multiple tx/rx queues, what happens is that multiple CPUs will trail each other on the same sequence of pools. The effect is that some allocations will end up costing O(100us) and more.

I encountered this as well: even with maxed out swiotlb memory the page_pool will suck a lot of pages from there. And TX allocations are left scrambling for scraps.

Why can't we create per device pools instead on relying on the swiotb?

Quoted 38 lines

I have tried to implement two improvements:
- a buddy allocator on top of each pool, so to make it quicker to find a candidate of the requested size
- make each CPU use a different sequence to explore other pools in case one is full, so they will not end up queueing one after the other
While they are very effective on the tails, for low load scenarios the current linear allocators is better. Thus this will take more investigation.

---
v1 -> v2:

- split components into separate commits
- simplified allocator, no need for a new page type
- many code cleanups
- also implement the rx side

Luigi Rizzo (5):
swiotlb: enforce pool nareas and nslabs invariants swiotlb/mm: Implement SWIOTLB nocopy page allocator net/swiotlb: Track bounce device per socket
net: Divert socket allocations to SWIOTLB for nocopy TX
swiotlb: Implement RX nocopy with fast recycling eviction

 drivers/base/core.c       |   1 +
 drivers/iommu/dma-iommu.c |   9 +-
 include/linux/netdevice.h |  21 +++
 include/linux/skbuff.h    |   7 +-
 include/linux/swiotlb.h   |  63 ++++++++
 include/net/sock.h        |  46 ++++++
 kernel/dma/direct.h       |  11 ++
 kernel/dma/swiotlb.c      | 296 ++++++++++++++++++++++++++++++++++++--
 mm/page_alloc.c           |  61 +++++++-
 net/core/page_pool.c      |  25 +++-
 net/core/sock.c           | 101 +++++++++++--
 11 files changed, 617 insertions(+), 24 deletions(-)
syzbot Tested-by [email protected] on the v2 series
#29 lore raw

[syzbot ci] Re: swiotlb: avoid swiotlb copy on network sockets

In reply to Luigi Rizzo
syzbot ci has tested the following series

syzbot ci has tested the following series

[v2] swiotlb: avoid swiotlb copy on network sockets
https://lore.kernel.org/all/[email protected]
* [PATCH v2 1/5] swiotlb: enforce pool nareas and nslabs invariants
* [PATCH v2 2/5] swiotlb/mm: Implement SWIOTLB nocopy page allocator
* [PATCH v2 3/5] net/swiotlb: Track bounce device per socket
* [PATCH v2 4/5] net: Divert socket allocations to SWIOTLB for nocopy TX
* [PATCH v2 5/5] swiotlb: Implement RX nocopy with fast recycling eviction

and found the following issue:

general protection fault in page_pool_put_unrefed_netmem

Full report is available here:
https://ci.syzbot.org/series/b17756c4-983f-4d7a-8415-3da06c0dda99

***

general protection fault in page_pool_put_unrefed_netmem
tree:      torvalds
URL:       https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux
base:      0a0d1d55dad570724bf8c7ea83409639cfb4be9b
arch:      amd64
compiler:  Debian clang version 22.1.8 (++20260613092233+e80beda6e255-1~exp1~20260613092250.77), Debian LLD 22.1.8
config:    https://ci.syzbot.org/builds/20e51ca2-88da-4674-9797-a62c64743095/config
syz repro: https://ci.syzbot.org/findings/4385dd7f-d1fc-4627-ad19-6862c780a90c/syz_repro
Oops: general protection fault, probably for non-canonical address 0xdffffc0000000082: 0000 [#1] SMP KASAN PTI
KASAN: null-ptr-deref in range [0x0000000000000410-0x0000000000000417]
CPU: 0 UID: 0 PID: 5836 Comm: syz.2.19 Not tainted syzkaller #0 PREEMPT(full) 
Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.2-debian-1.16.2-1 04/01/2014
RIP: 0010:swiotlb_addr_in_default_pool include/linux/swiotlb.h:177 [inline]
RIP: 0010:swiotlb_is_nocopy_addr include/linux/swiotlb.h:184 [inline]
RIP: 0010:__page_pool_put_page net/core/page_pool.c:851 [inline]
RIP: 0010:page_pool_put_unrefed_netmem+0x22a/0xd20 net/core/page_pool.c:930
Code: 5c 63 00 00 89 c5 31 ff 89 c6 e8 61 17 04 f8 85 ed 0f 84 07 08 00 00 e8 54 12 04 f8 49 81 c4 10 04 00 00 4c 89 e0 48 c1 e8 03 <42> 80 3c 28 00 74 08 4c 89 e7 e8 37 c7 73 f8 4d 8b 24 24 4d 85 e4
RSP: 0018:ffffc900039df230 EFLAGS: 00010206
RAX: 0000000000000082 RBX: 0000000000000000 RCX: ffff888160efda00
RDX: 0000000000000000 RSI: 0000000000000001 RDI: 0000000000000000
RBP: 0000000000000001 R08: ffffffff89c3aca5 R09: 0000000000000000
R10: 0000000000000000 R11: ffffffff8ed5a4e0 R12: 0000000000000410
R13: dffffc0000000000 R14: ffffea0004450300 R15: 0000000004450300
FS:  00007f08f83e86c0(0000) GS:ffff88818d6fc000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffd455a8fe8 CR3: 00000001134ac000 CR4: 00000000000006f0
Call Trace:
 <TASK>
 page_pool_put_netmem include/net/page_pool/helpers.h:343 [inline]
 page_pool_put_full_netmem include/net/page_pool/helpers.h:373 [inline]
 napi_pp_put_page+0x133/0x1e0 net/core/skbuff.c:1036
 skb_pp_recycle net/core/skbuff.c:1047 [inline]
 skb_free_head net/core/skbuff.c:1091 [inline]
 skb_release_data+0x8c3/0xab0 net/core/skbuff.c:1122
 skb_release_all net/core/skbuff.c:1197 [inline]
 __kfree_skb+0x5d/0x210 net/core/skbuff.c:1211
 llc_rcv+0x77a/0xcb0 net/llc/llc_input.c:-1
 __netif_receive_skb_list_ptype net/core/dev.c:6312 [inline]
 __netif_receive_skb_list_core+0x784/0x830 net/core/dev.c:6354
 __netif_receive_skb_list net/core/dev.c:6406 [inline]
 netif_receive_skb_list_internal+0x995/0xcf0 net/core/dev.c:6497
 netif_receive_skb_list+0x67/0x480 net/core/dev.c:6549
 xdp_recv_frames net/bpf/test_run.c:268 [inline]
 xdp_test_run_batch net/bpf/test_run.c:349 [inline]
 bpf_test_run_xdp_live+0x1875/0x1c20 net/bpf/test_run.c:378
 bpf_prog_test_run_xdp+0x7d8/0x11d0 net/bpf/test_run.c:1463
 bpf_prog_test_run+0x2c5/0x340 kernel/bpf/syscall.c:4804
 __sys_bpf+0xa22/0xd90 kernel/bpf/syscall.c:6385
 __do_sys_bpf kernel/bpf/syscall.c:6486 [inline]
 __se_sys_bpf kernel/bpf/syscall.c:6483 [inline]
 __x64_sys_bpf+0xba/0xd0 kernel/bpf/syscall.c:6483
 do_syscall_x64 arch/x86/entry/syscall_64.c:61 [inline]
 do_syscall_64+0x166/0x520 arch/x86/entry/syscall_64.c:84
 entry_SYSCALL_64_after_hwframe+0x77/0x7f
RIP: 0033:0x7f08f759e0d9
Code: ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 e8 ff ff ff f7 d8 64 89 01 48
RSP: 002b:00007f08f83e8028 EFLAGS: 00000246 ORIG_RAX: 0000000000000141
RAX: ffffffffffffffda RBX: 00007f08f7825fa0 RCX: 00007f08f759e0d9
RDX: 0000000000000048 RSI: 0000200000000600 RDI: 000000000000000a
RBP: 00007f08f7635024 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
R13: 00007f08f7826038 R14: 00007f08f7825fa0 R15: 00007fffc4c69778
 </TASK>
Modules linked in:
---[ end trace 0000000000000000 ]---
RIP: 0010:swiotlb_addr_in_default_pool include/linux/swiotlb.h:177 [inline]
RIP: 0010:swiotlb_is_nocopy_addr include/linux/swiotlb.h:184 [inline]
RIP: 0010:__page_pool_put_page net/core/page_pool.c:851 [inline]
RIP: 0010:page_pool_put_unrefed_netmem+0x22a/0xd20 net/core/page_pool.c:930
Code: 5c 63 00 00 89 c5 31 ff 89 c6 e8 61 17 04 f8 85 ed 0f 84 07 08 00 00 e8 54 12 04 f8 49 81 c4 10 04 00 00 4c 89 e0 48 c1 e8 03 <42> 80 3c 28 00 74 08 4c 89 e7 e8 37 c7 73 f8 4d 8b 24 24 4d 85 e4
RSP: 0018:ffffc900039df230 EFLAGS: 00010206
RAX: 0000000000000082 RBX: 0000000000000000 RCX: ffff888160efda00
RDX: 0000000000000000 RSI: 0000000000000001 RDI: 0000000000000000
RBP: 0000000000000001 R08: ffffffff89c3aca5 R09: 0000000000000000
R10: 0000000000000000 R11: ffffffff8ed5a4e0 R12: 0000000000000410
R13: dffffc0000000000 R14: ffffea0004450300 R15: 0000000004450300
FS:  00007f08f83e86c0(0000) GS:ffff88818d6fc000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007ffd455a8fe8 CR3: 00000001134ac000 CR4: 00000000000006f0
----------------
Code disassembly (best guess), 2 bytes skipped:
   0:	00 00                	add    %al,(%rax)
   2:	89 c5                	mov    %eax,%ebp
   4:	31 ff                	xor    %edi,%edi
   6:	89 c6                	mov    %eax,%esi
   8:	e8 61 17 04 f8       	call   0xf804176e
   d:	85 ed                	test   %ebp,%ebp
   f:	0f 84 07 08 00 00    	je     0x81c
  15:	e8 54 12 04 f8       	call   0xf804126e
  1a:	49 81 c4 10 04 00 00 	add    $0x410,%r12
  21:	4c 89 e0             	mov    %r12,%rax
  24:	48 c1 e8 03          	shr    $0x3,%rax
* 28:	42 80 3c 28 00       	cmpb   $0x0,(%rax,%r13,1) <-- trapping instruction
  2d:	74 08                	je     0x37
  2f:	4c 89 e7             	mov    %r12,%rdi
  32:	e8 37 c7 73 f8       	call   0xf873c76e
  37:	4d 8b 24 24          	mov    (%r12),%r12
  3b:	4d 85 e4             	test   %r12,%r12

***

If these findings have caused you to resend the series or submit a separate fix, please add the following tag to your commit message:

---
This report is generated by a bot. It may contain errors. syzbot ci engineers can be reached at [email protected].

To test a fix for this bug, please reply with #syz test (on a separate line) and attach the patch to the email.

Notes:
- The patch will be applied on top of the tested series (as an incremental fix).
- To test a new version of the whole series, please send it directly to [email protected].
- Arguments like custom git repos and branches are not supported.