net/mlx5e: Add support for PCIe congestion events
Subsystems (6)
Tariq Toukan
maintainerv3 cover net/mlx5e: Add support for PCIe congestion events
Hi,
This is V3. Find V2 here:
https://lore.kernel.org/all/[email protected]/
Find detailed feature description by Dragos below [1].
Regards,
Tariq
V3:
- Dropped sysfs configuration patch.
- Fixed compilation issue when !CONFIG_MLX5_CORE_EN.
- Updated cover letter description.
V2:
- Rebase on top of the IFC patches, they got pulled through mlx5-next.
[1]
PCIe congestion events are events generated by the firmware when the device side has sustained PCIe inbound or outbound traffic above certain thresholds. The high and low threshold are hysteresis thresholds to prevent flapping: once the high threshold has been reached, a low threshold event will be triggered only after the bandwidth usage went below the low threshold.
This series adds support for receiving and exposing such events as ethtool counters.
2 new pairs of counters are exposed: pci_bw_in/outbound_high/low. These should help the user understand if the device PCI is under pressure.
Planned followup patches:
- Allow configuration of thresholds through devlink.
- Add ethtool counter for wakeups which did not result in any state change.
Dragos Tatulea (2):
net/mlx5e: Create/destroy PCIe Congestion Event object net/mlx5e: Add device PCIe congestion ethtool stats
.../ethernet/mellanox/mlx5/counters.rst | 32 ++ .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 + .../mellanox/mlx5/core/en/pcie_cong_event.c | 315 ++++++++++++++++++ .../mellanox/mlx5/core/en/pcie_cong_event.h | 10 + .../net/ethernet/mellanox/mlx5/core/en_main.c | 3 + .../ethernet/mellanox/mlx5/core/en_stats.c | 1 + .../ethernet/mellanox/mlx5/core/en_stats.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/eq.c | 3 + .../ethernet/mellanox/mlx5/core/mlx5_core.h | 13 + 10 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.h
base-commit: 06baf9bfa6ca8db7d5f32e12e27d1dc1b7cb3a8a
Signature · 1 line
2.31.1
Tariq Toukan
maintainerPATCH v3 1/2 net/mlx5e: Create/destroy PCIe Congestion Event object
From: Dragos Tatulea <[email protected]>
Add initial infrastructure to create and destroy the PCIe Congestion Event object if the object is supported.
The verb for the object creation function is "set" instead of "create" because the function will accommodate the modify operation as well in a subsequent patch.
The next patches will hook it up to the event handler and will add actual functionality.
--- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 + .../mellanox/mlx5/core/en/pcie_cong_event.c | 140 ++++++++++++++++++ .../mellanox/mlx5/core/en/pcie_cong_event.h | 10 ++ .../net/ethernet/mellanox/mlx5/core/en_main.c | 3 + .../ethernet/mellanox/mlx5/core/mlx5_core.h | 13 ++ 6 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.h
drivers/net/ethernet/mellanox/mlx5/core/Makefile +1 -1
@@ -29,7 +29,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ - lib/crypto.o lib/sd.o + lib/crypto.o lib/sd.o en/pcie_cong_event.o # # Netdev extra
drivers/net/ethernet/mellanox/mlx5/core/en.h +2 -0
@@ -920,6 +920,8 @@ struct mlx5e_priv {struct notifier_block events_nb;struct notifier_block blocking_events_nb; +struct mlx5e_pcie_cong_event *cong_event; +struct udp_tunnel_nic_info nic_info;#ifdef CONFIG_MLX5_CORE_EN_DCB struct mlx5e_dcbx dcbx;
drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c new file +140 -0
140 lines not on this page · raw message
drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.h new file +10 -0
@@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. */ + +#ifndef __MLX5_PCIE_CONG_EVENT_H__ +#define __MLX5_PCIE_CONG_EVENT_H__ + +int mlx5e_pcie_cong_event_init (struct mlx5e_priv *priv); +void mlx5e_pcie_cong_event_cleanup (struct mlx5e_priv *priv); + +#endif /* __MLX5_PCIE_CONG_EVENT_H__ */
drivers/net/ethernet/mellanox/mlx5/core/en_main.c +3 -0
@@ -76,6 +76,7 @@#include "en/trap.h" #include "lib/devcom.h" #include "lib/sd.h" +#include "en/pcie_cong_event.h" static bool mlx5e_hw_gro_supported (struct mlx5_core_dev *mdev) {
@@ -5989,6 +5990,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)if (mlx5e_monitor_counter_supported (priv))mlx5e_monitor_counter_init (priv); +mlx5e_pcie_cong_event_init (priv);mlx5e_hv_vhca_stats_create (priv);if (netdev->reg_state !=NETREG_REGISTERED )return ;
@@ -6028,6 +6030,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv)mlx5e_nic_set_rx_mode (priv); +mlx5e_pcie_cong_event_cleanup (priv);mlx5e_hv_vhca_stats_destroy (priv);if (mlx5e_monitor_counter_supported (priv))mlx5e_monitor_counter_cleanup (priv);
drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +13 -0
@@ -495,4 +495,17 @@ static inline int mlx5_max_eq_cap_get(const struct mlx5_core_dev *dev) return 1 << MLX5_CAP_GEN(dev, log_max_eq); } + +static inline bool mlx5_pcie_cong_event_supported (struct mlx5_core_dev *dev) +{ +u64 features =MLX5_CAP_GEN_2_64 (dev, general_obj_types_127_64); + +if (!(features &MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_PCIE_CONG_EVENT )) +return false; + +if (dev->sd) +return false; + +return true; +}#endif /* __MLX5_CORE_H__ */
Signature · 1 line
2.31.1
Tariq Toukan
maintainerPATCH v3 2/2 net/mlx5e: Add device PCIe congestion ethtool stats
In reply to Tariq ToukanFrom: Dragos Tatulea <[email protected]>
Implement the PCIe Congestion Event notifier which triggers a work item to query the PCIe Congestion Event object. The result of the congestion state is reflected in the new ethtool stats:
* pci_bw_inbound_high: the device has crossed the high threshold for inbound PCIe traffic.
* pci_bw_inbound_low: the device has crossed the low threshold for inbound PCIe traffic
* pci_bw_outbound_high: the device has crossed the high threshold for outbound PCIe traffic.
* pci_bw_outbound_low: the device has crossed the low threshold for outbound PCIe traffic
The high and low thresholds are currently configured at 90% and 75%. These are hysteresis thresholds which help to check if the PCI bus on the device side is in a congested state.
If low + 1 = high then the device is in a congested state. If low == high then the device is not in a congested state.
The counters are also documented.
A follow-up patch will make the thresholds configurable.
--- .../ethernet/mellanox/mlx5/counters.rst | 32 ++++ .../mellanox/mlx5/core/en/pcie_cong_event.c | 175 ++++++++++++++++++ .../ethernet/mellanox/mlx5/core/en_stats.c | 1 + .../ethernet/mellanox/mlx5/core/en_stats.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/eq.c | 3 + 5 files changed, 212 insertions(+)
Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst +32 -0
@@ -1341,3 +1341,35 @@ Device Counters - The number of times the device owned queue had not enough buffers allocated. - Error + + * - `pci_bw_inbound_high` + - The number of times the device crossed the high inbound pcie bandwidth + threshold. To be compared to pci_bw_inbound_low to check if the device + is in a congested state. + If pci_bw_inbound_high == pci_bw_inbound_low then the device is not congested. + If pci_bw_inbound_high > pci_bw_inbound_low then the device is congested. + - Tnformative + + * - `pci_bw_inbound_low` + - The number of times the device crossed the low inbound PCIe bandwidth + threshold. To be compared to pci_bw_inbound_high to check if the device + is in a congested state. + If pci_bw_inbound_high == pci_bw_inbound_low then the device is not congested. + If pci_bw_inbound_high > pci_bw_inbound_low then the device is congested. + - Informative + + * - `pci_bw_outbound_high` + - The number of times the device crossed the high outbound pcie bandwidth + threshold. To be compared to pci_bw_outbound_low to check if the device + is in a congested state. + If pci_bw_outbound_high == pci_bw_outbound_low then the device is not congested. + If pci_bw_outbound_high > pci_bw_outbound_low then the device is congested. + - Informative + + * - `pci_bw_outbound_low` + - The number of times the device crossed the low outbound PCIe bandwidth + threshold. To be compared to pci_bw_outbound_high to check if the device + is in a congested state. + If pci_bw_outbound_high == pci_bw_outbound_low then the device is not congested. + If pci_bw_outbound_high > pci_bw_outbound_low then the device is congested. + - Informative
drivers/net/ethernet/mellanox/mlx5/core/en/pcie_cong_event.c +175 -0
225 lines not on this page · raw message
drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +1 -0
@@ -2612,6 +2612,7 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = {#ifdef CONFIG_MLX5_MACSEC &MLX5E_STATS_GRP (macsec_hw), #endif + &MLX5E_STATS_GRP(pcie_cong), };unsigned int mlx5e_nic_stats_grps_num (struct mlx5e_priv *priv)
drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +1 -0
@@ -535,5 +535,6 @@ extern MLX5E_DECLARE_STATS_GRP(ipsec_hw);extern MLX5E_DECLARE_STATS_GRP (ipsec_sw);extern MLX5E_DECLARE_STATS_GRP (ptp);extern MLX5E_DECLARE_STATS_GRP (macsec_hw); +extern MLX5E_DECLARE_STATS_GRP (pcie_cong);#endif /* __MLX5_EN_STATS_H__ */
drivers/net/ethernet/mellanox/mlx5/core/eq.c +3 -0
@@ -585,6 +585,9 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) async_event_mask |= (1ull <<MLX5_EVENT_TYPE_OBJECT_CHANGE ); +if (mlx5_pcie_cong_event_supported (dev)) + async_event_mask |= (1ull <<MLX5_EVENT_TYPE_OBJECT_CHANGE ); + mask[0 ] = async_event_mask;if (MLX5_CAP_GEN (dev, event_cap))
Signature · 1 line
2.31.1
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Quoted 8 lines · On Tue, 15 Jul 2025 17:30:19 +0300 you wrote:
Hi,
This is V3. Find V2 here:
https://lore.kernel.org/all/[email protected]/Find detailed feature description by Dragos below [1].
[...]
Here is the summary with links:
- [net-next,V3,1/2] net/mlx5e: Create/destroy PCIe Congestion Event object
https://git.kernel.org/netdev/net-next/c/ab2b0d4d6394 - [net-next,V3,2/2] net/mlx5e: Add device PCIe congestion ethtool stats https://git.kernel.org/netdev/net-next/c/8890ee6dcf6e
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html