RDMA/efa: Add AH usage counter with sysfs exposure
Subsystems (2)
PATCH 1/1 RDMA/efa: Add AH usage counter with sysfs exposure
Add tracking of unique Address Handle usage to provide visibility into active AH resource consumption. The implementation uses a hash table to deduplicate identical AH requests that receive the same handle, ensuring accurate resource counting.
The counter will be exposed via sysfs device attribute.
--- drivers/infiniband/hw/efa/Makefile | 4 +- drivers/infiniband/hw/efa/efa.h | 5 ++- drivers/infiniband/hw/efa/efa_main.c | 13 ++++++- drivers/infiniband/hw/efa/efa_sysfs.c | 33 ++++++++++++++++ drivers/infiniband/hw/efa/efa_sysfs.h | 15 +++++++ drivers/infiniband/hw/efa/efa_verbs.c | 56 ++++++++++++++++++++++++++- 6 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.c create mode 100644 drivers/infiniband/hw/efa/efa_sysfs.h
drivers/infiniband/hw/efa/Makefile +2 -2
@@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. # # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver. # obj-$(CONFIG_INFINIBAND_EFA) += efa.o -efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o +efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o efa_sysfs.o
drivers/infiniband/hw/efa/efa.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_
@@ -69,6 +69,9 @@ struct efa_dev {/* Only stores CQs with interrupts enabled */ struct xarray cqs_xa; +/* AH tracking xarray and counter*/ +struct xarray ahs_xa; +atomic64_t ah_count; };struct efa_ucontext {
drivers/infiniband/hw/efa/efa_main.c +12 -1
@@ -1,6 +1,6 @@// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* -* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/module.h>
@@ -12,6 +12,7 @@#include <rdma/uverbs_ioctl.h> #include "efa.h" +#include "efa_sysfs.h" #define PCI_DEV_ID_EFA0_VF 0xefa0#define PCI_DEV_ID_EFA1_VF 0xefa1
@@ -561,6 +562,8 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev) edev->dmadev = &pdev->dev; dev->pdev = pdev;xa_init (&dev->cqs_xa); +xa_init (&dev->ahs_xa); +atomic64_set (&dev->ah_count,0 ); pci_mem_bars =pci_select_bars (pdev,IORESOURCE_MEM );if (EFA_BASE_BAR_MASK & ~pci_mem_bars) {
@@ -619,8 +622,14 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)if (err) goto err_free_mgmnt_irq; + err =efa_sysfs_init (dev); +if (err) + goto err_admin_destroy; +return dev; +err_admin_destroy: +efa_com_admin_destroy (edev); err_free_mgmnt_irq:efa_free_irq (dev, &dev->admin_irq); err_disable_msix:
@@ -645,6 +654,7 @@ static void efa_remove_device(struct pci_dev *pdev,struct efa_com_dev *edev; edev = &dev->edev; +efa_sysfs_destroy (dev);efa_com_dev_reset (edev, reset_reason);efa_com_admin_destroy (edev);efa_free_irq (dev, &dev->admin_irq);
@@ -653,6 +663,7 @@ static void efa_remove_device(struct pci_dev *pdev,devm_iounmap (&pdev->dev, edev->reg_bar);efa_release_bars (dev,EFA_BASE_BAR_MASK );xa_destroy (&dev->cqs_xa); +xa_destroy (&dev->ahs_xa);ib_dealloc_device (&dev->ibdev);pci_disable_device (pdev); }
drivers/infiniband/hw/efa/efa_sysfs.c new file +33 -0
@@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. +*/ + +#include <linux/device.h> +#include <linux/sysfs.h> + +#include "efa_sysfs.h" + +static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count)); +} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" ); + +return 0 ; +} + +void efa_sysfs_destroy (struct efa_dev *dev) +{ +device_remove_file (&dev->pdev->dev, &dev_attr_ah_count); +}
drivers/infiniband/hw/efa/efa_sysfs.h new file +15 -0
@@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. +*/ + +#ifndef _EFA_SYSFS_H_ +#define _EFA_SYSFS_H_ + +#include "efa.h" + +int efa_sysfs_init (struct efa_dev *dev); + +void efa_sysfs_destroy (struct efa_dev *dev); + +#endif /* _EFA_SYSFS_H_ */
drivers/infiniband/hw/efa/efa_verbs.c +54 -2
91 lines not on this page · raw message
Signature · 1 line
2.47.3
Jason Gunthorpe
maintainerQuoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));
drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );
This is not the right way to use sysfs in rdma drivers.
Also we have netlink counters as the prefered approach why are you using sysfs?
Jason
Quoted 2 lines · Tom Sela
Add tracking of unique Address Handle usage to provide visibility into active AH resource consumption. The implementation uses a hash table to
It seems that xa is used instead of hash table in your commit?
Zhu Yanjun
Quoted 277 lines
deduplicate identical AH requests that receive the same handle, ensuring accurate resource counting.
The counter will be exposed via sysfs device attribute.
Reviewed-by Michael Margolin <[email protected]>Reviewed-by Yonatan Nachum <[email protected]>Signed-off-by Tom Sela <[email protected]>--- drivers/infiniband/hw/efa/Makefile | 4 +- drivers/infiniband/hw/efa/efa.h | 5 ++- drivers/infiniband/hw/efa/efa_main.c | 13 ++++++- drivers/infiniband/hw/efa/efa_sysfs.c | 33 ++++++++++++++++ drivers/infiniband/hw/efa/efa_sysfs.h | 15 +++++++ drivers/infiniband/hw/efa/efa_verbs.c | 56 ++++++++++++++++++++++++++- 6 files changed, 120 insertions(+), 6 deletions(-)create mode 100644
drivers/infiniband/hw/efa/efa_sysfs.ccreate mode 100644drivers/infiniband/hw/efa/efa_sysfs.hdrivers/infiniband/hw/efa/Makefile +2
-2@@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. # # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver. # obj-$(CONFIG_INFINIBAND_EFA) += efa.o -efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o +efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o efa_sysfs.odrivers/infiniband/hw/efa/efa.h +4
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -69,6 +69,9 @@ struct efa_dev {/* Only stores CQs with interrupts enabled */ struct xarray cqs_xa; +/* AH tracking xarray and counter*/ +struct xarray ahs_xa; +atomic64_t ah_count; };struct efa_ucontext {drivers/infiniband/hw/efa/efa_main.c +12
-1@@ -1,6 +1,6 @@// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* -* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/module.h> @@ -12,6 +12,7 @@#include <rdma/uverbs_ioctl.h> #include "efa.h" +#include "efa_sysfs.h" #define PCI_DEV_ID_EFA0_VF 0xefa0#define PCI_DEV_ID_EFA1_VF 0xefa1@@ -561,6 +562,8 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev) edev->dmadev = &pdev->dev; dev->pdev = pdev;xa_init (&dev->cqs_xa); +xa_init (&dev->ahs_xa); +atomic64_set (&dev->ah_count,0 ); pci_mem_bars =pci_select_bars (pdev,IORESOURCE_MEM );if (EFA_BASE_BAR_MASK & ~pci_mem_bars) {@@ -619,8 +622,14 @@ static struct efa_dev *efa_probe_device(struct pci_dev *pdev)if (err) goto err_free_mgmnt_irq; + err =efa_sysfs_init (dev); +if (err) + goto err_admin_destroy; +return dev; +err_admin_destroy: +efa_com_admin_destroy (edev); err_free_mgmnt_irq:efa_free_irq (dev, &dev->admin_irq); err_disable_msix:@@ -645,6 +654,7 @@ static void efa_remove_device(struct pci_dev *pdev,struct efa_com_dev *edev; edev = &dev->edev; +efa_sysfs_destroy (dev);efa_com_dev_reset (edev, reset_reason);efa_com_admin_destroy (edev);efa_free_irq (dev, &dev->admin_irq);@@ -653,6 +663,7 @@ static void efa_remove_device(struct pci_dev *pdev,devm_iounmap (&pdev->dev, edev->reg_bar);efa_release_bars (dev,EFA_BASE_BAR_MASK );xa_destroy (&dev->cqs_xa); +xa_destroy (&dev->ahs_xa);ib_dealloc_device (&dev->ibdev);pci_disable_device (pdev); }drivers/infiniband/hw/efa/efa_sysfs.c new file +33
-0@@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. +*/ + +#include <linux/device.h> +#include <linux/sysfs.h> + +#include "efa_sysfs.h" + +static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count)); +} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" ); + +return 0 ; +} + +void efa_sysfs_destroy (struct efa_dev *dev) +{ +device_remove_file (&dev->pdev->dev, &dev_attr_ah_count); +}drivers/infiniband/hw/efa/efa_sysfs.h new file +15
-0@@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. +*/ + +#ifndef _EFA_SYSFS_H_ +#define _EFA_SYSFS_H_ + +#include "efa.h" + +int efa_sysfs_init (struct efa_dev *dev); + +void efa_sysfs_destroy (struct efa_dev *dev); + +#endif /* _EFA_SYSFS_H_ */ drivers/infiniband/hw/efa/efa_verbs.c +54
-2@@ -1,6 +1,6 @@// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* -* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include <linux/dma-buf.h> @@ -2104,6 +2104,51 @@ int efa_mmap(struct ib_ucontext *ibucontext,return __efa_mmap (dev, ucontext, vma); } +static int efa_add_ah (struct efa_dev *dev,u16 ah) +{ +unsigned long refcount; +void *entry; +int err; + +xa_lock (&dev->ahs_xa); + entry =xa_load (&dev->ahs_xa, ah); + refcount = entry ?xa_to_value (entry) :0 ; +if (refcount ==0 ) +atomic64_inc (&dev->ah_count); + + err =xa_err (__xa_store (&dev->ahs_xa, ah,xa_mk_value (refcount +1 ),GFP_ATOMIC )); +xa_unlock (&dev->ahs_xa); + +return err; +} + +static int efa_remove_ah (struct efa_dev *dev,u16 ah) +{ +unsigned long refcount; +void *entry; +int err; + +xa_lock (&dev->ahs_xa); + entry =xa_load (&dev->ahs_xa, ah); + refcount = entry ?xa_to_value (entry) :0 ; +if (refcount ==0 ) { +/* AH already removed or never existed - unexpected but handle gracefully */ +xa_unlock (&dev->ahs_xa); +return 0 ; + } + + refcount--; + +if (refcount ==0 ) { + err =xa_err (__xa_erase (&dev->ahs_xa, ah)); +atomic64_dec (&dev->ah_count); + }else { + err =xa_err (__xa_store (&dev->ahs_xa, ah,xa_mk_value (refcount),GFP_ATOMIC )); + } +xa_unlock (&dev->ahs_xa); +return err; +} +static int efa_ah_destroy (struct efa_dev *dev,struct efa_ah *ah) {struct efa_com_destroy_ah_params params = {@@ -2150,6 +2195,10 @@ int efa_create_ah(struct ib_ah *ibah,memcpy (ah->id, ah_attr->grh.dgid.raw,sizeof (ah->id)); ah->ah = result.ah; + err =efa_add_ah (dev, ah->ah); +if (err) + goto err_destroy_ah; + resp.efa_address_handle = result.ah;if (udata->outlen) {@@ -2158,13 +2207,15 @@ int efa_create_ah(struct ib_ah *ibah,if (err) {ibdev_dbg (&dev->ibdev,"Failed to copy udata for create_ah response\n" ); - goto err_destroy_ah; + goto err_remove_ah; } }ibdev_dbg (&dev->ibdev,"Created ah[%d]\n" , ah->ah);return 0 ; +err_remove_ah: +efa_remove_ah (dev, ah->ah); err_destroy_ah:efa_ah_destroy (dev, ah); err_out:@@ -2185,6 +2236,7 @@ int efa_destroy_ah(struct ib_ah *ibah, u32 flags)return -EOPNOTSUPP ; } +efa_remove_ah (dev, ah->ah);efa_ah_destroy (dev, ah);return 0 ; }
Gal Pressman
reviewerdrivers/infiniband/hw/efa/efa_verbs.c int efa_mmap(struct ib_ucontext *ibucontext, · line 2117
Quoted 11 lines · Tom Sela
drivers/infiniband/hw/efa/efa_verbs.c +11
-0+static int efa_add_ah (struct efa_dev *dev,u16 ah) +{ +unsigned long refcount; +void *entry; +int err; + +xa_lock (&dev->ahs_xa); + entry =xa_load (&dev->ahs_xa, ah); + refcount = entry ?xa_to_value (entry) :0 ; +if (refcount ==0 ) +atomic64_inc (&dev->ah_count);
What happens to this increment if the store fails?
Quoted 6 lines
drivers/infiniband/hw/efa/efa_verbs.c +6
-0+ + err =xa_err (__xa_store (&dev->ahs_xa, ah,xa_mk_value (refcount +1 ),GFP_ATOMIC )); +xa_unlock (&dev->ahs_xa); + +return err; +}
Gal Pressman
reviewerQuoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );This is not the right way to use sysfs in rdma drivers.
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
BTW, isn't this something that can be added to restrack?
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );This is not the right way to use sysfs in rdma drivers.
Quoted 6 lines
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
BTW, isn't this something that can be added to restrack?
Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.
Thanks.
Michael Margolin
maintainerQuoted 26 lines · Leon Romanovsky
Quoted 25 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );This is not the right way to use sysfs in rdma drivers.
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
Quoted 8 lines
Quoted 1 line
BTW, isn't this something that can be added to restrack?
Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.
Thanks.
This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.
In a followup series we will suggest netlink counters extension to support driver specific resources.
Michael
Quoted 34 lines · Michael Margolin
Quoted 33 lines · Leon Romanovsky
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );This is not the right way to use sysfs in rdma drivers.
Quoted 6 lines
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
BTW, isn't this something that can be added to restrack?
Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.
Quoted 8 lines
Quoted 1 line
Thanks.
This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
Thanks
Quoted 3 lines
Michael
Gal Pressman
reviewerQuoted 2 lines · Leon Romanovsky
Quoted 2 lines
Stats also doesn't seem as the right place
for this.
Because?
Quoted 10 lines
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
I don't understand, how do you use bpftrace for this use case?
Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines
Stats also doesn't seem as the right place
for this.Because?
Quoted 8 lines
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
Quoted 8 lines
Quoted 2 lines
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
I don't understand, how do you use bpftrace for this use case?
Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.
Their proposed counter can be implemented by counting calls to efa_com_create_ah minus calls to efa_com_destroy_ah.
You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter
Thanks
Quoted 1 line
Michael Margolin
maintainerQuoted 38 lines · Leon Romanovsky
Quoted 37 lines · Michael Margolin
Quoted 34 lines · Leon Romanovsky
Quoted 20 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
drivers/infiniband/hw/efa/efa_sysfs.c new file +5
-0+static ssize_t ah_count_show (struct device *dev,struct device_attribute *attr,char *buf) +{ +struct efa_dev *efa_dev =pci_get_drvdata (to_pci_dev (dev)); + +return sysfs_emit (buf,"%lld\n" ,atomic64_read (&efa_dev->ah_count));drivers/infiniband/hw/efa/efa_sysfs.c line 25
Quoted 10 lines
drivers/infiniband/hw/efa/efa_sysfs.c new file +10
-0+} + +static DEVICE_ATTR_RO (ah_count); + +int efa_sysfs_init (struct efa_dev *dev) +{ +struct device *device = &dev->pdev->dev; + +if (device_create_file (device, &dev_attr_ah_count)) +dev_err (device,"Failed to create AH count sysfs file\n" );This is not the right way to use sysfs in rdma drivers.
Quoted 6 lines
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
BTW, isn't this something that can be added to restrack?
Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.
Thanks.
Quoted 8 lines
Quoted 5 lines
This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.
How can the kernel and this new counter report a different number of AH objects?
When application creates multiple AH objects for same peer, the device reuses the existing resource and returns its AH number. The new counter counts unique AH numbers thus represents the amount of device resources currently in use.
Quoted 7 lines
Quoted 4 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
This isn't intended for internal debug use but is aimed for end users to allow tracking of a limited device resource usage by their applications.
Michael
Quoted 45 lines · Michael Margolin
Quoted 44 lines · Leon Romanovsky
Quoted 42 lines · Michael Margolin
Quoted 26 lines · Leon Romanovsky
Quoted 25 lines · Gal Pressman
Quoted 19 lines · Jason Gunthorpe
Quoted 5 lines · Tom Sela
(quoted diff) +5
-0+static ssize_t ah_count_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct efa_dev *efa_dev = pci_get_drvdata(to_pci_dev(dev)); + + return sysfs_emit(buf, "%lld\n", atomic64_read(&efa_dev->ah_count));Review
Quoted 10 lines
(quoted diff) +10
-0+} + +static DEVICE_ATTR_RO(ah_count); + +int efa_sysfs_init(struct efa_dev *dev) +{ + struct device *device = &dev->pdev->dev; + + if (device_create_file(device, &dev_attr_ah_count)) + dev_err(device, "Failed to create AH count sysfs file\n");This is not the right way to use sysfs in rdma drivers.
Quoted 2 lines
Also we have netlink counters as the prefered approach why are you using sysfs?
Yes, and EFA already supports stats reporting, the sysfs choice is strange..
Quoted 8 lines
Quoted 1 line
BTW, isn't this something that can be added to restrack?
Unlikely. Most drivers that implement such counters were written long before bpftrace became widely used. I don't think modern drivers should carry these counters, as they are trivial to collect without requiring any kernel changes. This is especially true for EFA, which does not support kverbs.
Thanks.
This approach was selected since this case doesn't naturally fit any of the suggested ideas. It represents usage level of device AH objects which might be different than the number of kernel objects as usually covered by restrack count. Stats also doesn't seem as the right place for this.
Quoted 7 lines
Quoted 2 lines
How can the kernel and this new counter report a different number of AH objects?
When application creates multiple AH objects for same peer, the device reuses the existing resource and returns its AH number. The new counter counts unique AH numbers thus represents the amount of device resources currently in use.
https://lore.kernel.org/all/20260215175707.GC12989@unreal
Thanks
Michael Margolin
maintainerQuoted 24 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines
Stats also doesn't seem as the right place
for this.Because?
Quoted 8 lines
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
Quoted 8 lines
Quoted 2 lines
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
I don't understand, how do you use bpftrace for this use case?
Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.
Quoted 7 lines
Their proposed counter can be implemented by counting calls to
efa_com_create_ahminus calls toefa_com_destroy_ah.You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counter
Calls to efa_com_create_ah minus calls to efa_com_destroy_ah will not always result in correct number of consumed device resources as multiple calls to efa_com_create_ah can return the same AH number.
Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.
Michael
Quoted 28 lines · Michael Margolin
Quoted 27 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines
Stats also doesn't seem as the right place
for this.Because?
Quoted 8 lines
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
Quoted 8 lines
Quoted 2 lines
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
I don't understand, how do you use bpftrace for this use case?
Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.
Their proposed counter can be implemented by counting calls to
efa_com_create_ahminus calls toefa_com_destroy_ah.
Quoted 8 lines
Quoted 4 lines
You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counterCalls to
efa_com_create_ahminus calls toefa_com_destroy_ahwill not always result in correct number of consumed device resources as multiple calls toefa_com_create_ahcan return the same AH number.
bpftrace supports map and can count unique ids.
Quoted 5 lines
Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.
BPF doesn't require any kernel rebuild. It works out-of-the-box on even old kernels.
Thanks
Quoted 3 lines
Michael
Michael Margolin
maintainerQuoted 41 lines · Leon Romanovsky
Quoted 28 lines · Michael Margolin
Quoted 27 lines · Leon Romanovsky
Quoted 14 lines · Gal Pressman
Quoted 2 lines · Leon Romanovsky
Quoted 2 lines
Stats also doesn't seem as the right place
for this.Because?
Quoted 8 lines
How can the kernel and this new counter report a different number of AH objects?
Quoted 3 lines
In a followup series we will suggest netlink counters extension to support driver specific resources.
Quoted 8 lines
Quoted 2 lines
bpftrace is generally the right tool, unless you can detail why it does not fit your specific debugging scenario.
I don't understand, how do you use bpftrace for this use case?
Once you get to debug a system in a certain state, bpftrace won't help you see events that happened in the past. You won't be able to know how many AH were created.
Their proposed counter can be implemented by counting calls to
efa_com_create_ahminus calls toefa_com_destroy_ah.Quoted 8 lines
Quoted 4 lines
You have two ways to get it:
1. run bfptrace with your reproducer
2. check FW to get their internal counterCalls to
efa_com_create_ahminus calls toefa_com_destroy_ahwill not always result in correct number of consumed device resources as multiple calls toefa_com_create_ahcan return the same AH number.bpftrace supports map and can count unique ids.
Quoted 1 line
Quoted 7 lines
Quoted 4 lines
Additionally we are looking to expose this info to customers without requiring a kernel rebuild or the use of debug tools, similar to how device and port statistics can be read in sysfs or through the rdma tool.
BPF doesn't require any kernel rebuild. It works out-of-the-box on even old kernels.
I'll try to give a higher-level overview of the need.
I don't argue that bpftrace is a powerful debug tool that can be used to collect various info from the kernel, including this one and other data that we expose through sysfs, netlink, and rdma-core. This being said, I don't think it should be considered as a replacement for an explicit and stable kernel ABI.
What the proposed change is trying to do is give end users the ability to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter). Such user processes can't depend on in-kernel function names or any internal logic that might change from time to time, and expect well-defined and solid interfaces. I don't see how bpftrace can serve this need.
Michael
Jason Gunthorpe
maintainerQuoted 3 lines · Michael Margolin
to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).
Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'
and the value you want is the simple difference?
Jason
Quoted 10 lines · Jason Gunthorpe
Quoted 3 lines · Michael Margolin
to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).
Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'and the value you want is the simple difference?
Which can be collected from FW through FWCTL?
I don't super excited to see slow, unique sysfs UAPI field in RDMA subsystem. They are interested to get FW information, let's use interfaces which are intended for it.
Thanks
Quoted 2 lines
Jason
Michael Margolin
maintainerQuoted 11 lines · Leon Romanovsky
Quoted 10 lines · Jason Gunthorpe
Quoted 3 lines · Michael Margolin
to monitor AH usage in production. Resource usage and traffic counters are usually collected periodically (every 1-5 seconds) by dedicated collectors (e.g., Prometheus node exporter).
Can you just have two simple stats
'# HW AHs created'
'# HW AHs destroyed'and the value you want is the simple difference?
I think that adding two separate counters as you suggest under /sys/class/infiniband/*/hw_counters/ can work for us. Will submit a new patch for this option.
Quoted 8 lines
Which can be collected from FW through FWCTL?
I don't super excited to see slow, unique sysfs UAPI field in RDMA subsystem. They are interested to get FW information, let's use interfaces which are intended for it.
Thanks
Leon, the device currently doesn't expose this info and it's generated in the driver.
Michael