Skip to contentSkip to conversation

RDMA/efa: Add AH usage counter with sysfs exposure

PATCH In review Tom Sela 6 participants 18 messages Feb 11 – Feb 18

Subsystems (2)
lore
Share & tools
lore

Tom Sela

author
#0 lore raw

PATCH 1/1 RDMA/efa: Add AH usage counter with sysfs exposure

Add tracking of unique Address Handle usage to provide visibility into active AH resource consumption. The implementation uses a hash table to deduplicate identical AH requests that receive the same handle, ensuring accurate resource counting.

The counter will be exposed via sysfs device attribute.

---
 drivers/infiniband/hw/efa/Makefile    |  4 +-
 drivers/infiniband/hw/efa/efa.h       |  5 ++-
 drivers/infiniband/hw/efa/efa_main.c  | 13 ++++++-
 drivers/infiniband/hw/efa/efa_sysfs.c | 33 ++++++++++++++++
 drivers/infiniband/hw/efa/efa_sysfs.h | 15 +++++++
 drivers/infiniband/hw/efa/efa_verbs.c | 56 ++++++++++++++++++++++++++-
 6 files changed, 120 insertions(+), 6 deletions(-)
 create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.c
 create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.h
drivers/infiniband/hw/efa/Makefile +2 -2
index 6e83083af0bc..0ba04eab17a1 100644
@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
 #
 # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver.
 #
 
 obj-$(CONFIG_INFINIBAND_EFA) += efa.o
 
-efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o
+efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o efa_sysfs.o
drivers/infiniband/hw/efa/efa.h +4 -1
index 96f9c3bc98b2..d332bc4edcb7 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -69,6 +69,9 @@ struct efa_dev {
 
 	/* Only stores CQs with interrupts enabled */
 	struct xarray cqs_xa;
+	/* AH tracking xarray and counter*/
+	struct xarray ahs_xa;
+	atomic64_t ah_count;
 };
 
 struct efa_ucontext {
drivers/infiniband/hw/efa/efa_main.c +12 -1
index 6c415b9adb5f..3c6fa5af941a 100644
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <linux/module.h>
@@ -12,6 +12,7 @@
 #include <rdma/uverbs_ioctl.h>
 
 #include "efa.h"
+#include "efa_sysfs.h"
 
 #define PCI_DEV_ID_EFA0_VF 0xefa0
 #define PCI_DEV_ID_EFA1_VF 0xefa1
@@ -561,6 +562,8 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
 	edev->dmadev = &pdev->dev;
 	dev->pdev = pdev;
 	xa_init(&dev->cqs_xa);
+	xa_init(&dev->ahs_xa);
+	atomic64_set(&dev->ah_count, 0);
 
 	pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);
 	if (EFA_BASE_BAR_MASK & ~pci_mem_bars) {
@@ -619,8 +622,14 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
 	if (err)
 		goto err_free_mgmnt_irq;
 
+	err = efa_sysfs_init(dev);
+	if (err)
+		goto err_admin_destroy;
+
 	return dev;
 
+err_admin_destroy:
+	efa_com_admin_destroy(edev);
 err_free_mgmnt_irq:
 	efa_free_irq(dev, &dev->admin_irq);
 err_disable_msix:
@@ -645,6 +654,7 @@ static void efa_remove_device(struct pci_dev *pdev,
 	struct efa_com_dev *edev;
 
 	edev = &dev->edev;
+	efa_sysfs_destroy(dev);
 	efa_com_dev_reset(edev, reset_reason);
 	efa_com_admin_destroy(edev);
 	efa_free_irq(dev, &dev->admin_irq);
@@ -653,6 +663,7 @@ static void efa_remove_device(struct pci_dev *pdev,
 	devm_iounmap(&pdev->dev, edev->reg_bar);
 	efa_release_bars(dev, EFA_BASE_BAR_MASK);
 	xa_destroy(&dev->cqs_xa);
+	xa_destroy(&dev->ahs_xa);
 	ib_dealloc_device(&dev->ibdev);
 	pci_disable_device(pdev);
 }
drivers/infiniband/hw/efa/efa_sysfs.c new file +33 -0
new file mode 100644
index 000000000000..79602cf77424
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <linux/device.h>
+#include <linux/sysfs.h>
+
+#include "efa_sysfs.h"
+
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");
+
+	return 0;
+}
+
+void efa_sysfs_destroy(struct efa_dev *dev)
+{
+	device_remove_file(&dev->pdev->dev, &dev_attr_ah_count);
+}
drivers/infiniband/hw/efa/efa_sysfs.h new file +15 -0
new file mode 100644
index 000000000000..fda3a885c150
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_SYSFS_H_
+#define _EFA_SYSFS_H_
+
+#include "efa.h"
+
+int efa_sysfs_init(struct efa_dev *dev);
+
+void efa_sysfs_destroy(struct efa_dev *dev);
+
+#endif /* _EFA_SYSFS_H_ */
drivers/infiniband/hw/efa/efa_verbs.c +54 -2

91 lines not on this page · raw message

Signature · 1 line
2.47.3
maintainer
#1 lore raw
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Also we have netlink counters as the prefered approach why are you using sysfs?

Jason

reviewer
#2 lore raw
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

BTW, isn't this something that can be added to restrack?

#3 lore raw
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 6 lines
Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

BTW, isn't this something that can be added to restrack?

Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.

Thanks.

maintainer
#4 lore raw
Quoted 26 lines · Leon Romanovsky
Quoted 25 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

Quoted 8 lines
Quoted 1 line

BTW, isn't this something that can be added to restrack?

Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.

Thanks.

This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.

In a followup series we will suggest netlink counters extension to support driver specific resources.

Michael

#5 lore raw
Quoted 34 lines · Michael Margolin
Quoted 33 lines · Leon Romanovsky
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 6 lines
Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

BTW, isn't this something that can be added to restrack?

Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.

Quoted 8 lines
Quoted 1 line

Thanks.

This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

Thanks

Quoted 3 lines

Michael

reviewer
#6 lore raw
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines

Stats also doesn't seem as the right place
for this.

Because?

Quoted 10 lines

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

I don't understand, how do you use bpftrace for this use case?

Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.

#7 lore raw
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines

Stats also doesn't seem as the right place
for this.

Because?

Quoted 8 lines

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

Quoted 8 lines
Quoted 2 lines

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

I don't understand, how do you use bpftrace for this use case?

Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.

Their proposed counter can be implemented by counting calls to efa_com_create_ah minus calls to efa_com_destroy_ah.

You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter

Thanks

Quoted 1 line
maintainer
#8 lore raw
Quoted 24 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines

Stats also doesn't seem as the right place
for this.

Because?

Quoted 8 lines

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

Quoted 8 lines
Quoted 2 lines

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

I don't understand, how do you use bpftrace for this use case?

Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.

Quoted 7 lines

Their proposed counter can be implemented by counting calls to efa_com_create_ah minus calls to efa_com_destroy_ah.

You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter

Calls to efa_com_create_ah minus calls to efa_com_destroy_ah will not always result in correct number of consumed device resources as multiple calls to efa_com_create_ah can return the same AH number.

Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.

Michael

#9 lore raw
Quoted 28 lines · Michael Margolin
Quoted 27 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines

Stats also doesn't seem as the right place
for this.

Because?

Quoted 8 lines

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

Quoted 8 lines
Quoted 2 lines

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

I don't understand, how do you use bpftrace for this use case?

Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.

Their proposed counter can be implemented by counting calls to efa_com_create_ah minus calls to efa_com_destroy_ah.

Quoted 8 lines
Quoted 4 lines

You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter

Calls to efa_com_create_ah minus calls to efa_com_destroy_ah will not always result in correct number of consumed device resources as multiple calls to efa_com_create_ah can return the same AH number.

bpftrace supports map and can count unique ids.

Quoted 5 lines

Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.

BPF doesn't require any kernel rebuild. It works out-of-the-box on even old kernels.

Thanks

Quoted 3 lines

Michael

maintainer
#10 lore raw
Quoted 41 lines · Leon Romanovsky
Quoted 28 lines · Michael Margolin
Quoted 27 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines

Stats also doesn't seem as the right place
for this.

Because?

Quoted 8 lines

How can the kernel and this new counter report a different number of AH objects?

Quoted 3 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

Quoted 8 lines
Quoted 2 lines

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

I don't understand, how do you use bpftrace for this use case?

Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.

Their proposed counter can be implemented by counting calls to efa_com_create_ah minus calls to efa_com_destroy_ah.

Quoted 8 lines
Quoted 4 lines

You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter

Calls to efa_com_create_ah minus calls to efa_com_destroy_ah will not always result in correct number of consumed device resources as multiple calls to efa_com_create_ah can return the same AH number.

bpftrace supports map and can count unique ids.

Quoted 1 line
Quoted 7 lines
Quoted 4 lines

Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.

BPF doesn't require any kernel rebuild. It works out-of-the-box on even old kernels.

I'll try to give a higher-level overview of the need.

I don't argue that bpftrace is a powerful debug tool that can be used to collect various info from the kernel, including this one and other data that we expose through sysfs, netlink, and rdma-core. This being said, I don't think it should be considered as a replacement for an explicit and stable kernel ABI.

What the proposed change is trying to do is give end users the ability to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter). Such user processes can't depend on in-kernel function names or any internal logic that might change from time to time, and expect well-defined and solid interfaces. I don't see how bpftrace can serve this need.

Michael

maintainer
#11 lore raw
Quoted 3 lines · Michael Margolin

to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).

Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'

and the value you want is the simple difference?

Jason

#12 lore raw
Quoted 10 lines · Jason Gunthorpe
Quoted 3 lines · Michael Margolin

to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).

Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'

and the value you want is the simple difference?

Which can be collected from FW through FWCTL?

I don't super excited to see slow, unique sysfs UAPI field in RDMA subsystem. They are interested to get FW information, let's use interfaces which are intended for it.

Thanks

Quoted 2 lines

Jason

maintainer
#13 lore raw
Quoted 11 lines · Leon Romanovsky
Quoted 10 lines · Jason Gunthorpe
Quoted 3 lines · Michael Margolin

to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).

Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'

and the value you want is the simple difference?

I think that adding two separate counters as you suggest under /sys/class/infiniband/*/hw_counters/ can work for us. Will submit a new patch for this option.

Quoted 8 lines

Which can be collected from FW through FWCTL?

I don't super excited to see slow, unique sysfs UAPI field in RDMA subsystem. They are interested to get FW information, let's use interfaces which are intended for it.

Thanks

Leon, the device currently doesn't expose this info and it's generated in the driver.

Michael

maintainer
#14 lore raw
In reply to Leon Romanovsky
Quoted 38 lines · Leon Romanovsky
Quoted 37 lines · Michael Margolin
Quoted 34 lines · Leon Romanovsky
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

drivers/infiniband/hw/efa/efa_sysfs.c line 25

Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 6 lines
Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

BTW, isn't this something that can be added to restrack?

Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.

Thanks.

Quoted 8 lines
Quoted 5 lines

This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.

How can the kernel and this new counter report a different number of AH objects?

When application creates multiple AH objects for same peer, the device reuses the existing resource and returns its AH number. The new counter counts unique AH numbers thus represents the amount of device resources currently in use.

Quoted 7 lines
Quoted 4 lines

In a followup series we will suggest netlink counters extension to support driver specific resources.

bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.

This isn't intended for internal debug use but is aimed for end users to allow tracking of a limited device resource usage by their applications.

Michael

#15 lore raw
Quoted 45 lines · Michael Margolin
Quoted 44 lines · Leon Romanovsky
Quoted 42 lines · Michael Margolin
Quoted 26 lines · Leon Romanovsky
Quoted 25 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
(quoted diff) +5 -0
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));

Review

Quoted 10 lines
(quoted diff) +10 -0
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");

This is not the right way to use sysfs in rdma drivers.

Quoted 2 lines

Also we have netlink counters as the prefered approach why are you using sysfs?

Yes, and EFA already supports stats reporting, the sysfs choice is strange..

Quoted 8 lines
Quoted 1 line

BTW, isn't this something that can be added to restrack?

Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.

Thanks.

This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.

Quoted 7 lines
Quoted 2 lines

How can the kernel and this new counter report a different number of AH objects?

When application creates multiple AH objects for same peer, the device reuses the existing resource and returns its AH number. The new counter counts unique AH numbers thus represents the amount of device resources currently in use.

https://lore.kernel.org/all/20260215175707.GC12989@unreal

Thanks

#16 lore raw
In reply to Tom Sela
Quoted 2 lines · Tom Sela

Add tracking of unique Address Handle usage to provide visibility into active AH resource consumption. The implementation uses a hash table to

It seems that xa is used instead of hash table in your commit?

Zhu Yanjun

Quoted 277 lines

deduplicate identical AH requests that receive the same handle, ensuring accurate resource counting.

The counter will be exposed via sysfs device attribute.

---
  drivers/infiniband/hw/efa/Makefile    |  4 +-
  drivers/infiniband/hw/efa/efa.h       |  5 ++-
  drivers/infiniband/hw/efa/efa_main.c  | 13 ++++++-
  drivers/infiniband/hw/efa/efa_sysfs.c | 33 ++++++++++++++++
  drivers/infiniband/hw/efa/efa_sysfs.h | 15 +++++++
  drivers/infiniband/hw/efa/efa_verbs.c | 56 ++++++++++++++++++++++++++-
  6 files changed, 120 insertions(+), 6 deletions(-)

create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.c create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.h

drivers/infiniband/hw/efa/Makefile +2 -2
index 6e83083af0bc..0ba04eab17a1 100644
@@ -1,9 +1,9 @@
  # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  #
  # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver.
  #
 
  obj-$(CONFIG_INFINIBAND_EFA) += efa.o
 
-efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o
+efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o efa_sysfs.o
drivers/infiniband/hw/efa/efa.h +4 -1
index 96f9c3bc98b2..d332bc4edcb7 100644
@@ -1,6 +1,6 @@
  /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
  /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
   */
 
  #ifndef _EFA_H_
@@ -69,6 +69,9 @@ struct efa_dev {
 
  	/* Only stores CQs with interrupts enabled */
  	struct xarray cqs_xa;
+	/* AH tracking xarray and counter*/
+	struct xarray ahs_xa;
+	atomic64_t ah_count;
  };
 
  struct efa_ucontext {
drivers/infiniband/hw/efa/efa_main.c +12 -1
index 6c415b9adb5f..3c6fa5af941a 100644
@@ -1,6 +1,6 @@
  // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
  /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
   */
 
  #include <linux/module.h>
@@ -12,6 +12,7 @@
  #include <rdma/uverbs_ioctl.h>
 
  #include "efa.h"
+#include "efa_sysfs.h"
 
  #define PCI_DEV_ID_EFA0_VF 0xefa0
  #define PCI_DEV_ID_EFA1_VF 0xefa1
@@ -561,6 +562,8 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
  	edev->dmadev = &pdev->dev;
  	dev->pdev = pdev;
  	xa_init(&dev->cqs_xa);
+	xa_init(&dev->ahs_xa);
+	atomic64_set(&dev->ah_count, 0);
 
  	pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);
  	if (EFA_BASE_BAR_MASK & ~pci_mem_bars) {
@@ -619,8 +622,14 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
  	if (err)
  		goto err_free_mgmnt_irq;
 
+	err = efa_sysfs_init(dev);
+	if (err)
+		goto err_admin_destroy;
+
  	return dev;
 
+err_admin_destroy:
+	efa_com_admin_destroy(edev);
  err_free_mgmnt_irq:
  	efa_free_irq(dev, &dev->admin_irq);
  err_disable_msix:
@@ -645,6 +654,7 @@ static void efa_remove_device(struct pci_dev *pdev,
  	struct efa_com_dev *edev;
 
  	edev = &dev->edev;
+	efa_sysfs_destroy(dev);
  	efa_com_dev_reset(edev, reset_reason);
  	efa_com_admin_destroy(edev);
  	efa_free_irq(dev, &dev->admin_irq);
@@ -653,6 +663,7 @@ static void efa_remove_device(struct pci_dev *pdev,
  	devm_iounmap(&pdev->dev, edev->reg_bar);
  	efa_release_bars(dev, EFA_BASE_BAR_MASK);
  	xa_destroy(&dev->cqs_xa);
+	xa_destroy(&dev->ahs_xa);
  	ib_dealloc_device(&dev->ibdev);
  	pci_disable_device(pdev);
  }
drivers/infiniband/hw/efa/efa_sysfs.c new file +33 -0
new file mode 100644
index 000000000000..79602cf77424
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <linux/device.h>
+#include <linux/sysfs.h>
+
+#include "efa_sysfs.h"
+
+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev));
+
+	return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));
+}
+
+static DEVICE_ATTR_RO(ah_count);
+
+int efa_sysfs_init(struct efa_dev *dev)
+{
+	struct device *device = &dev->pdev->dev;
+
+	if (device_create_file(device, &dev_attr_ah_count))
+		dev_err(device, "Failed to create AH count sysfs file\n");
+
+	return 0;
+}
+
+void efa_sysfs_destroy(struct efa_dev *dev)
+{
+	device_remove_file(&dev->pdev->dev, &dev_attr_ah_count);
+}
drivers/infiniband/hw/efa/efa_sysfs.h new file +15 -0
new file mode 100644
index 000000000000..fda3a885c150
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_SYSFS_H_
+#define _EFA_SYSFS_H_
+
+#include "efa.h"
+
+int efa_sysfs_init(struct efa_dev *dev);
+
+void efa_sysfs_destroy(struct efa_dev *dev);
+
+#endif /* _EFA_SYSFS_H_ */
drivers/infiniband/hw/efa/efa_verbs.c +54 -2
index 22d3e25c3b9d..1d8cb4a7f946 100644
@@ -1,6 +1,6 @@
  // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  /*
- * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
   */
 
  #include <linux/dma-buf.h>
@@ -2104,6 +2104,51 @@ int efa_mmap(struct ib_ucontext *ibucontext,
  	return __efa_mmap(dev, ucontext, vma);
  }
 
+static int efa_add_ah(struct efa_dev *dev, u16 ah)
+{
+	unsigned long refcount;
+	void *entry;
+	int err;
+
+	xa_lock(&dev->ahs_xa);
+	entry = xa_load(&dev->ahs_xa, ah);
+	refcount = entry ? xa_to_value(entry) : 0;
+	if (refcount == 0)
+		atomic64_inc(&dev->ah_count);
+
+	err = xa_err(__xa_store(&dev->ahs_xa, ah, xa_mk_value(refcount + 1), GFP_ATOMIC));
+	xa_unlock(&dev->ahs_xa);
+
+	return err;
+}
+
+static int efa_remove_ah(struct efa_dev *dev, u16 ah)
+{
+	unsigned long refcount;
+	void *entry;
+	int err;
+
+	xa_lock(&dev->ahs_xa);
+	entry = xa_load(&dev->ahs_xa, ah);
+	refcount = entry ? xa_to_value(entry) : 0;
+	if (refcount == 0) {
+		/* AH already removed or never existed - unexpected but handle gracefully */
+		xa_unlock(&dev->ahs_xa);
+		return 0;
+	}
+
+	refcount--;
+
+	if (refcount == 0) {
+		err = xa_err(__xa_erase(&dev->ahs_xa, ah));
+		atomic64_dec(&dev->ah_count);
+	} else {
+		err = xa_err(__xa_store(&dev->ahs_xa, ah, xa_mk_value(refcount), GFP_ATOMIC));
+	}
+	xa_unlock(&dev->ahs_xa);
+	return err;
+}
+
  static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)
  {
  	struct efa_com_destroy_ah_params params = {
@@ -2150,6 +2195,10 @@ int efa_create_ah(struct ib_ah *ibah,
  	memcpy(ah->id, ah_attr->grh.dgid.raw, sizeof(ah->id));
  	ah->ah = result.ah;
 
+	err = efa_add_ah(dev, ah->ah);
+	if (err)
+		goto err_destroy_ah;
+
  	resp.efa_address_handle = result.ah;
 
  	if (udata->outlen) {
@@ -2158,13 +2207,15 @@ int efa_create_ah(struct ib_ah *ibah,
  		if (err) {
  			ibdev_dbg(&dev->ibdev,
  				  "Failed to copy udata for create_ah response\n");
-			goto err_destroy_ah;
+			goto err_remove_ah;
  		}
  	}
  	ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah);
 
  	return 0;
 
+err_remove_ah:
+	efa_remove_ah(dev, ah->ah);
  err_destroy_ah:
  	efa_ah_destroy(dev, ah);
  err_out:
@@ -2185,6 +2236,7 @@ int efa_destroy_ah(struct ib_ah *ibah, u32 flags)
  		return -EOPNOTSUPP;
  	}
 
+	efa_remove_ah(dev, ah->ah);
  	efa_ah_destroy(dev, ah);
  	return 0;
  }
reviewer
#17 lore raw
In reply to Tom Sela

drivers/infiniband/hw/efa/efa_verbs.c int efa_mmap(struct ib_ucontext *ibucontext, · line 2117

Quoted 11 lines · Tom Sela
drivers/infiniband/hw/efa/efa_verbs.c +11 -0
+static int efa_add_ah(struct efa_dev *dev, u16 ah)
+{
+	unsigned long refcount;
+	void *entry;
+	int err;
+
+	xa_lock(&dev->ahs_xa);
+	entry = xa_load(&dev->ahs_xa, ah);
+	refcount = entry ? xa_to_value(entry) : 0;
+	if (refcount == 0)
+		atomic64_inc(&dev->ah_count);

What happens to this increment if the store fails?

Quoted 6 lines
drivers/infiniband/hw/efa/efa_verbs.c +6 -0
+
+	err = xa_err(__xa_store(&dev->ahs_xa, ah, xa_mk_value(refcount + 1), GFP_ATOMIC));
+	xa_unlock(&dev->ahs_xa);
+
+	return err;
+}