Skip to contentSkip to conversation

net: devmem: add netdev_has_dmabuf_binding() helper

PATCH In review Dragos Tatulea 3 participants 8 messages Aug 10 – Aug 11 2 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 net: devmem: add netdev_has_dmabuf_binding() helper

Currently there is no way to check if there is a dmabuf bound on a TX queue. RX side has netif_rxq_has_unreadable_mp() which gives a hint of it.

To help with that, this patch add a helper to check whether a devmem dmabuf binding is active on a given netdev, optionally filtered by the DMA device the dmabuf was mapped against.

This API is necessary for the upcoming support of data direct in mlx5e to allow blocking accidental swapping of DMA devices while devmem is active.

---
 include/net/netmem.h | 19 +++++++++++++++++++
 net/core/devmem.c    | 20 ++++++++++++++++++++
 2 files changed, 39 insertions(+)
include/net/netmem.h +19 -0
index bccacd21b6c3..0e332e04ea16 100644
@@ -363,16 +363,35 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
 	return netmem_to_nmdesc(netmem)->dma_addr;
 }
 
+struct net_device;
+
 #if defined(CONFIG_NET_DEVMEM)
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return niov->type == NET_IOV_DMABUF;
 }
+
+/**
+ * netdev_has_dmabuf_binding - is there a dmabuf binding for a
+ * given @dev and @dma_dev
+ *
+ * @dev: netdev to check bindings
+ * @dma_dev: dma device to check, when NULL it will not be checked
+ *
+ * Return: true if a binding is found under @dev and possibily @dma_dev
+ */
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev);
 #else
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return false;
 }
+
+static inline bool netdev_has_dmabuf_binding(struct net_device *dev,
+					     struct device *dma_dev)
+{
+	return false;
+}
 #endif
 
 void __get_netmem(netmem_ref netmem);
net/core/devmem.c +20 -0
index 957d6b96216b..f13a0e343684 100644
@@ -364,6 +364,26 @@ struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
 	return binding;
 }
 
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;
+
+	rcu_read_lock();
+	xa_for_each(&net_devmem_dmabuf_bindings, id, binding) {
+		if (READ_ONCE(binding->dev) == dev &&
+		    (!dma_dev || binding->attachment->dev == dma_dev)) {
+			found = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return found;
+}
+EXPORT_SYMBOL_GPL(netdev_has_dmabuf_binding);
+
 void net_devmem_get_net_iov(struct net_iov *niov)
 {
 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
Signature · 1 line
2.54.0
#1 lore raw
Quoted 62 lines · Dragos Tatulea

Currently there is no way to check if there is a dmabuf bound on a TX queue. RX side has netif_rxq_has_unreadable_mp() which gives a hint of it.

To help with that, this patch add a helper to check whether a devmem dmabuf binding is active on a given netdev, optionally filtered by the DMA device the dmabuf was mapped against.

This API is necessary for the upcoming support of data direct in mlx5e to allow blocking accidental swapping of DMA devices while devmem is active.

---
 include/net/netmem.h | 19 +++++++++++++++++++
 net/core/devmem.c    | 20 ++++++++++++++++++++
 2 files changed, 39 insertions(+)
include/net/netmem.h +19 -0
index bccacd21b6c3..0e332e04ea16 100644
@@ -363,16 +363,35 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
 	return netmem_to_nmdesc(netmem)->dma_addr;
 }
 
+struct net_device;
+
 #if defined(CONFIG_NET_DEVMEM)
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return niov->type == NET_IOV_DMABUF;
 }
+
+/**
+ * netdev_has_dmabuf_binding - is there a dmabuf binding for a
+ * given @dev and @dma_dev
+ *
+ * @dev: netdev to check bindings
+ * @dma_dev: dma device to check, when NULL it will not be checked
+ *
+ * Return: true if a binding is found under @dev and possibily @dma_dev
+ */
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev);
 #else
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return false;
 }
+
+static inline bool netdev_has_dmabuf_binding(struct net_device *dev,
+					     struct device *dma_dev)
+{
+	return false;
+}
 #endif
 
 void __get_netmem(netmem_ref netmem);
net/core/devmem.c +0 -0
index 957d6b96216b..f13a0e343684 100644

net/core/devmem.c struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) · line 371

Quoted 8 lines
net/core/devmem.c +5 -0
@@ -364,6 +364,26 @@ struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
 	return binding;
 }
 
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;

xmas tree. Other than that LGTM. Presumably it's better to ship via your upcoming series to show the actual usage?

#2 lore raw
Quoted 66 lines · Stanislav Fomichev
Quoted 65 lines · Dragos Tatulea

Currently there is no way to check if there is a dmabuf bound on a TX queue. RX side has netif_rxq_has_unreadable_mp() which gives a hint of it.

To help with that, this patch add a helper to check whether a devmem dmabuf binding is active on a given netdev, optionally filtered by the DMA device the dmabuf was mapped against.

This API is necessary for the upcoming support of data direct in mlx5e to allow blocking accidental swapping of DMA devices while devmem is active.

---
 include/net/netmem.h | 19 +++++++++++++++++++
 net/core/devmem.c    | 20 ++++++++++++++++++++
 2 files changed, 39 insertions(+)
include/net/netmem.h +19 -0
index bccacd21b6c3..0e332e04ea16 100644
@@ -363,16 +363,35 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
 	return netmem_to_nmdesc(netmem)->dma_addr;
 }
 
+struct net_device;
+
 #if defined(CONFIG_NET_DEVMEM)
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return niov->type == NET_IOV_DMABUF;
 }
+
+/**
+ * netdev_has_dmabuf_binding - is there a dmabuf binding for a
+ * given @dev and @dma_dev
+ *
+ * @dev: netdev to check bindings
+ * @dma_dev: dma device to check, when NULL it will not be checked
+ *
+ * Return: true if a binding is found under @dev and possibily @dma_dev
+ */
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev);
 #else
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
 	return false;
 }
+
+static inline bool netdev_has_dmabuf_binding(struct net_device *dev,
+					     struct device *dma_dev)
+{
+	return false;
+}
 #endif
 
 void __get_netmem(netmem_ref netmem);
net/core/devmem.c +0 -0
index 957d6b96216b..f13a0e343684 100644
@@ -364,6 +364,26 @@ struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
 	return binding;
 }
 
Quoted 7 lines

net/core/devmem.c struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) · line 371

Quoted 5 lines
net/core/devmem.c +5 -0
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;

xmas tree. Other than that LGTM.

Seems like Bobby found a valid corner case that needs to be handled.

Quoted 1 line

Presumably it's better to ship via your upcoming series to show the actual usage?

I will do that eventually, but would like to get some ACKs about this bit so that Tariq doesn't have to re-send the whole series every time.

Thanks,
Dragos

A message that is not in the archive (sent off-list or not yet received): [email protected]

#4 lore raw
Quoted 25 lines · bobbyeshleman

On Mon, Aug 10, 2026 at 08:54:45PM +0300, Dragos Tatulea wrote:> [...]

Quoted 24 lines
net/core/devmem.c +20 -0
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;
+
+	rcu_read_lock();
+	xa_for_each(&net_devmem_dmabuf_bindings, id, binding) {
+		if (READ_ONCE(binding->dev) == dev &&
+		    (!dma_dev || binding->attachment->dev == dma_dev)) {
+			found = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return found;
+}
+EXPORT_SYMBOL_GPL(netdev_has_dmabuf_binding);
+
 void net_devmem_get_net_iov(struct net_iov *niov)
 {
 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
Quoted 9 lines
Quoted 2 lines

2.54.0

I think netdev_has_dmabuf_binding() can return false even when there exists one or more in-flight skbs (e.g., in sk_write_queue) backed by the txq binding (won't technically be inactive until the final reference is dropped and the binding is freed)? If so, can this still detect when it is safe to swap the underlying dma dev?

Oh, good point! I didn't realize that the binding is refcounted and can outlive its xarray slot.

Thanks,
Dragos

#5 lore raw
Quoted 37 lines · Dragos Tatulea
Quoted 25 lines · bobbyeshleman

On Mon, Aug 10, 2026 at 08:54:45PM +0300, Dragos Tatulea wrote:> [...]

Quoted 24 lines
net/core/devmem.c +20 -0
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;
+
+	rcu_read_lock();
+	xa_for_each(&net_devmem_dmabuf_bindings, id, binding) {
+		if (READ_ONCE(binding->dev) == dev &&
+		    (!dma_dev || binding->attachment->dev == dma_dev)) {
+			found = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return found;
+}
+EXPORT_SYMBOL_GPL(netdev_has_dmabuf_binding);
+
 void net_devmem_get_net_iov(struct net_iov *niov)
 {
 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
Quoted 9 lines
Quoted 2 lines

2.54.0

I think netdev_has_dmabuf_binding() can return false even when there exists one or more in-flight skbs (e.g., in sk_write_queue) backed by the txq binding (won't technically be inactive until the final reference is dropped and the binding is freed)? If so, can this still detect when it is safe to swap the underlying dma dev?

Quoted 3 lines

Oh, good point! I didn't realize that the binding is refcounted and can outlive its xarray slot.

I can only think of adding another list that tracks bindings for their lifetime or between xarray erasure and deletion. Not too happy about it though.

There's also the possibility of iterating over dmabufs and their attachment. But this requires a function in dmabuf to scan the dmabufs and attachments under the internal lock similar to dma_buf_debug_show(). But I don't think that will be accepted. So this is even worse.

Thanks,
Dragos

A message that is not in the archive (sent off-list or not yet received): [email protected]

#7 lore raw
Quoted 52 lines · bobbyeshleman
Quoted 51 lines · Dragos Tatulea
Quoted 37 lines · Dragos Tatulea
Quoted 25 lines · bobbyeshleman

On Mon, Aug 10, 2026 at 08:54:45PM +0300, Dragos Tatulea wrote:> [...]

Quoted 24 lines
net/core/devmem.c +20 -0
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+	struct net_devmem_dmabuf_binding *binding;
+	unsigned long id;
+	bool found = false;
+
+	rcu_read_lock();
+	xa_for_each(&net_devmem_dmabuf_bindings, id, binding) {
+		if (READ_ONCE(binding->dev) == dev &&
+		    (!dma_dev || binding->attachment->dev == dma_dev)) {
+			found = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return found;
+}
+EXPORT_SYMBOL_GPL(netdev_has_dmabuf_binding);
+
 void net_devmem_get_net_iov(struct net_iov *niov)
 {
 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
Quoted 9 lines
Quoted 2 lines

2.54.0

I think netdev_has_dmabuf_binding() can return false even when there exists one or more in-flight skbs (e.g., in sk_write_queue) backed by the txq binding (won't technically be inactive until the final reference is dropped and the binding is freed)? If so, can this still detect when it is safe to swap the underlying dma dev?

Quoted 3 lines

Oh, good point! I didn't realize that the binding is refcounted and can outlive its xarray slot.

I can only think of adding another list that tracks bindings for their lifetime or between xarray erasure and deletion. Not too happy about it though.

There's also the possibility of iterating over dmabufs and their attachment. But this requires a function in dmabuf to scan the dmabufs and attachments under the internal lock similar to dma_buf_debug_show(). But I don't think that will be accepted. So this is even worse.

Quoted 9 lines
Quoted 2 lines

Thanks,
Dragos

I agree, those options aren't great.

Would it be possible to drop these skbs (based on some comparison with binding->attachment->dev) in validate_xmit_unreadable_skb()? Not sure if we have enough context at that point in the TX path?

Even if we would, I wouldn't do it...

Quoted 3 lines

tbh I'd lean towards your idea of adding a new lifetime list, if there isn't a better way.

Ok. Will implement the lifetime list (or xarray) if I can't come up with something better until then.

Thanks,
Dragos

#8 lore raw
In reply to Dragos Tatulea
Quoted 63 lines · Dragos Tatulea

Currently there is no way to check if there is a dmabuf bound on a TX queue. RX side has netif_rxq_has_unreadable_mp() which gives a hint of it.

To help with that, this patch add a helper to check whether a devmem dmabuf binding is active on a given netdev, optionally filtered by the DMA device the dmabuf was mapped against.

This API is necessary for the upcoming support of data direct in mlx5e to allow blocking accidental swapping of DMA devices while devmem is active.

---
 include/net/netmem.h | 19 +++++++++++++++++++
 net/core/devmem.c    | 20 ++++++++++++++++++++
 2 files changed, 39 insertions(+)
include/net/netmem.h +19 -0
index bccacd21b6c3..0e332e04ea16 100644
@@ -363,16 +363,35 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
        return netmem_to_nmdesc(netmem)->dma_addr;
 }
 
+struct net_device;
+
 #if defined(CONFIG_NET_DEVMEM)
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
        return niov->type == NET_IOV_DMABUF;
 }
+
+/**
+ * netdev_has_dmabuf_binding - is there a dmabuf binding for a
+ * given @dev and @dma_dev
+ *
+ * @dev: netdev to check bindings
+ * @dma_dev: dma device to check, when NULL it will not be checked
+ *
+ * Return: true if a binding is found under @dev and possibily @dma_dev
+ */
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev);
 #else
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
        return false;
 }
+
+static inline bool netdev_has_dmabuf_binding(struct net_device *dev,
+                                            struct device *dma_dev)
+{
+       return false;
+}
 #endif
 
 void __get_netmem(netmem_ref netmem);
net/core/devmem.c +0 -0
index 957d6b96216b..f13a0e343684 100644

net/core/devmem.c struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) · line 373

Quoted 10 lines
net/core/devmem.c +7 -0
@@ -364,6 +364,26 @@ struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
        return binding;
 }
 
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+       struct net_devmem_dmabuf_binding *binding;
+       unsigned long id;
+       bool found = false;
+
+       rcu_read_lock();

Why do you need rcu locking? I thought xarrays handle concurrency inside their helpers.

Signature · 2 lines
Thanks,
Mina
#9 lore raw
Quoted 67 lines · Mina Almasry
Quoted 66 lines · Dragos Tatulea

Currently there is no way to check if there is a dmabuf bound on a TX queue. RX side has netif_rxq_has_unreadable_mp() which gives a hint of it.

To help with that, this patch add a helper to check whether a devmem dmabuf binding is active on a given netdev, optionally filtered by the DMA device the dmabuf was mapped against.

This API is necessary for the upcoming support of data direct in mlx5e to allow blocking accidental swapping of DMA devices while devmem is active.

---
 include/net/netmem.h | 19 +++++++++++++++++++
 net/core/devmem.c    | 20 ++++++++++++++++++++
 2 files changed, 39 insertions(+)
include/net/netmem.h +19 -0
index bccacd21b6c3..0e332e04ea16 100644
@@ -363,16 +363,35 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
        return netmem_to_nmdesc(netmem)->dma_addr;
 }
 
+struct net_device;
+
 #if defined(CONFIG_NET_DEVMEM)
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
        return niov->type == NET_IOV_DMABUF;
 }
+
+/**
+ * netdev_has_dmabuf_binding - is there a dmabuf binding for a
+ * given @dev and @dma_dev
+ *
+ * @dev: netdev to check bindings
+ * @dma_dev: dma device to check, when NULL it will not be checked
+ *
+ * Return: true if a binding is found under @dev and possibily @dma_dev
+ */
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev);
 #else
 static inline bool net_is_devmem_iov(const struct net_iov *niov)
 {
        return false;
 }
+
+static inline bool netdev_has_dmabuf_binding(struct net_device *dev,
+                                            struct device *dma_dev)
+{
+       return false;
+}
 #endif
 
 void __get_netmem(netmem_ref netmem);
net/core/devmem.c +0 -0
index 957d6b96216b..f13a0e343684 100644
@@ -364,6 +364,26 @@ struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
        return binding;
 }
 
Quoted 11 lines

net/core/devmem.c struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) · line 373

Quoted 7 lines
net/core/devmem.c +7 -0
+bool netdev_has_dmabuf_binding(struct net_device *dev, struct device *dma_dev)
+{
+       struct net_devmem_dmabuf_binding *binding;
+       unsigned long id;
+       bool found = false;
+
+       rcu_read_lock();

Why do you need rcu locking? I thought xarrays handle concurrency inside their helpers.

Not needed. Monkey see monkey do from net_devmem_lookup_dmabuf(). Will drop.

Thank,
Dragos