ethtool: linkstate: add a statistic for PHY down events
Subsystems (6)
Jakub Kicinski
maintainerPATCH v2 1/1 ethtool: linkstate: add a statistic for PHY down events
The previous attempt to augment carrier_down (see Link) was not met with much enthusiasm so let's do the simple thing of exposing what some devices already maintain. Add a common ethtool statistic for link going down. Currently users have to maintain per-driver mapping to extract the right stat from the vendor-specific ethtool -S stats. carrier_down does not fit the bill because it counts a lot of software related false positives.
Add the statistic to the extended link state API to steer vendors towards implementing all of it.
Implement for bnxt and all Linux-controlled PHYs. mlx5 and (possibly) enic also have a counter for this but I leave the implementation to their maintainers.
---
The Linux PHY part is compile-tested only. Could anyone (who's not at netdev conf?) help testing it?
v2:
- add the support for all Linux-driven PHYs
- LinkDownEvents -> link_down_events
v1: https://lore.kernel.org/all/[email protected]/
---
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
CC: [email protected]
--- Documentation/networking/ethtool-netlink.rst | 1 + .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 15 +++++++++++ drivers/net/phy/phy.c | 1 + include/linux/ethtool.h | 14 +++++++++++ include/linux/phy.h | 3 +++ include/uapi/linux/ethtool_netlink.h | 2 ++ net/ethtool/linkstate.c | 25 ++++++++++++++++++- 7 files changed, 60 insertions(+), 1 deletion(-)
Documentation/networking/ethtool-netlink.rst +1 -0
@@ -491,6 +491,7 @@ any attributes. ``ETHTOOL_A_LINKSTATE_SQI_MAX`` u32 Max support SQI value ``ETHTOOL_A_LINKSTATE_EXT_STATE`` u8 link extended state ``ETHTOOL_A_LINKSTATE_EXT_SUBSTATE`` u8 link extended substate + ``ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT`` u64 count of link down events ==================================== ====== ============================ For most NIC drivers, the value of ``ETHTOOL_A_LINKSTATE_LINK`` returns
drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +15 -0
@@ -4112,6 +4112,20 @@ static void bnxt_get_rmon_stats(struct net_device *dev, *ranges = bnxt_rmon_ranges; } +static void bnxt_get_link_ext_stats (struct net_device *dev, +struct ethtool_link_ext_stats *stats) +{ +struct bnxt *bp =netdev_priv (dev); +u64 *rx; + +if (BNXT_VF (bp) || !(bp->flags &BNXT_FLAG_PORT_STATS_EXT )) +return ; + + rx = bp->rx_port_stats_ext.sw_stats; + stats->link_down_events = + *(rx +BNXT_RX_STATS_EXT_OFFSET (link_down_events)); +} +void bnxt_ethtool_free (struct bnxt *bp) {kfree (bp->test_info);
@@ -4161,6 +4175,7 @@ const struct ethtool_ops bnxt_ethtool_ops = { .get_eeprom = bnxt_get_eeprom, .set_eeprom = bnxt_set_eeprom, .get_link = bnxt_get_link, + .get_link_ext_stats = bnxt_get_link_ext_stats, .get_eee = bnxt_get_eee, .set_eee = bnxt_set_eee, .get_module_info = bnxt_get_module_info,
drivers/net/phy/phy.c +1 -0
@@ -67,6 +67,7 @@ static void phy_link_down(struct phy_device *phydev) { phydev->phy_link_change (phydev, false);phy_led_trigger_change_speed (phydev); +WRITE_ONCE (phydev->link_down_events, phydev->link_down_events +1 ); }static const char *phy_pause_str (struct phy_device *phydev)
include/linux/ethtool.h +14 -0
@@ -125,6 +125,17 @@ struct ethtool_link_ext_state_info { }; }; +struct ethtool_link_ext_stats { +/* Custom Linux statistic for PHY level link down events. +* In a simpler world it should be equal to netdev->carrier_down_count +* unfortunately netdev also counts local reconfigurations which don't +* actually take the physical link down, not to mention NC-SI which, +* if present, keeps the link up regardless of host state. +* This statistic counts when PHY _actually_ went down, or lost link. +*/ +u64 link_down_events; +}; +/** * ethtool_rxfh_indir_default - get default value for RX flow hash indirection * @index: Index in RX flow hash indirection table
@@ -481,6 +492,7 @@ struct ethtool_module_power_mode_params {* do not attach ext_substate attribute to netlink message). If link_ext_state * and link_ext_substate are unknown, return -ENODATA. If not implemented, * link_ext_state and link_ext_substate will not be sent to userspace. +* @get_link_ext_stats: Read extra link-related counters. * @get_eeprom_len: Read range of EEPROM addresses for validation of * @get_eeprom and @set_eeprom requests. * Returns 0 if device does not support EEPROM access.
@@ -652,6 +664,8 @@ struct ethtool_ops {u32 (*get_link)(struct net_device *);int (*get_link_ext_state)(struct net_device *,struct ethtool_link_ext_state_info *); +void (*get_link_ext_stats)(struct net_device *, +struct ethtool_link_ext_stats *);int (*get_eeprom_len)(struct net_device *);int (*get_eeprom)(struct net_device *,struct ethtool_eeprom *,u8 *);
include/linux/phy.h +3 -0
@@ -600,6 +600,7 @@ struct macsec_ops;* @psec: Pointer to Power Sourcing Equipment control struct * @lock: Mutex for serialization access to PHY * @state_queue: Work queue for state machine +* @link_down_events: Number of times link was lost * @shared: Pointer to private data shared by phys in one package * @priv: Pointer to driver private data *
@@ -723,6 +724,8 @@ struct phy_device {int pma_extable; +unsigned int link_down_events; +void (*phy_link_change)(struct phy_device *phydev,bool up);void (*adjust_link)(struct net_device *dev);
include/uapi/linux/ethtool_netlink.h +2 -0
@@ -262,6 +262,8 @@ enum {ETHTOOL_A_LINKSTATE_SQI_MAX ,/* u32 */ ETHTOOL_A_LINKSTATE_EXT_STATE ,/* u8 */ ETHTOOL_A_LINKSTATE_EXT_SUBSTATE ,/* u8 */ +ETHTOOL_A_LINKSTATE_PAD , +ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT ,/* u64 */ /* add new constants above here */ __ETHTOOL_A_LINKSTATE_CNT,
net/ethtool/linkstate.c +24 -1
@@ -13,6 +13,7 @@ struct linkstate_reply_data {int link;int sqi;int sqi_max; +struct ethtool_link_ext_stats link_stats;bool link_ext_state_provided;struct ethtool_link_ext_state_info ethtool_link_ext_state_info; };
@@ -22,7 +23,7 @@ struct linkstate_reply_data {const struct nla_policy ethnl_linkstate_get_policy[] = { [ETHTOOL_A_LINKSTATE_HEADER ] = -NLA_POLICY_NESTED (ethnl_header_policy), +NLA_POLICY_NESTED (ethnl_header_policy_stats), };static int linkstate_get_sqi (struct net_device *dev)
@@ -107,6 +108,19 @@ static int linkstate_prepare_data(const struct ethnl_req_info *req_base, goto out; } +ethtool_stats_init ((u64 *)&data->link_stats, +sizeof (data->link_stats) /8 ); + +if (req_base->flags ÐTOOL_FLAG_STATS ) { +if (dev->phydev) + data->link_stats.link_down_events = +READ_ONCE (dev->phydev->link_down_events); + +if (dev->ethtool_ops->get_link_ext_stats) + dev->ethtool_ops->get_link_ext_stats (dev, + &data->link_stats); + } + ret =0 ; out:ethnl_ops_complete (dev);
@@ -134,6 +148,9 @@ static int linkstate_reply_size(const struct ethnl_req_info *req_base,if (data->ethtool_link_ext_state_info.__link_ext_substate) len +=nla_total_size (sizeof (u8));/* LINKSTATE_EXT_SUBSTATE */ +if (data->link_stats.link_down_events !=ETHTOOL_STAT_NOT_SET ) + len +=nla_total_size_64bit (sizeof (u64)); +return len; }
@@ -166,6 +183,12 @@ static int linkstate_fill_reply(struct sk_buff *skb,return -EMSGSIZE ; } +if (data->link_stats.link_down_events !=ETHTOOL_STAT_NOT_SET ) +if (nla_put_u64_64bit (skb,ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT , + data->link_stats.link_down_events, +ETHTOOL_A_LINKSTATE_PAD )) +return -EMSGSIZE ; +return 0 ; }
Signature · 1 line
2.37.3
Quoted 12 lines · Jakub Kicinski
The previous attempt to augment
carrier_down(see Link) was not met with much enthusiasm so let's do the simple thing of exposing what some devices already maintain. Add a common ethtool statistic for link going down. Currently users have to maintain per-driver mapping to extract the right stat from the vendor-specific ethtool -S stats.carrier_downdoes not fit the bill because it counts a lot of software related false positives.Add the statistic to the extended link state API to steer vendors towards implementing all of it.
Quoted 7 lines
Implement for bnxt and all Linux-controlled PHYs. mlx5 and (possibly) enic also have a counter for this but I leave the implementation to their maintainers.
Signed-off-by Jakub Kicinski <[email protected]>---
[snip]
Looks good just one nit/comment, feel free to respin or add:
include/linux/phy.h struct phy_device { · line 727
Quoted 5 lines
include/linux/phy.h +1
-0@@ -723,6 +724,8 @@ struct phy_device {int pma_extable; +unsigned int link_down_events;
Should not this be an u64 to match what the extended link state can report? Not that I would hope that anyone had a chance to witness 4 billion link down events using PHYLIB.
Signature · 1 line
Florian
Jakub Kicinski
maintainerQuoted 9 lines · On Thu, 27 Oct 2022 19:50:35 -0700 Florian Fainelli wrote:
include/linux/phy.h struct phy_device { · line 727
Quoted 5 lines
include/linux/phy.h +1
-0@@ -723,6 +724,8 @@ struct phy_device {int pma_extable; +unsigned int link_down_events;Should not this be an u64 to match what the extended link state can report? Not that I would hope that anyone had a chance to witness 4 billion link down events using PHYLIB.
How about I also make the uAPI side 32 bit? I made it 64b because that feels like the default these days but as you say, re-training a link 4 billion times is unlikely to happen in one person's lifetime :S
Andrew Lunn
maintainerdrivers/net/phy/phy.c static void phy_link_down(struct phy_device *phydev) · line 70
Quoted 5 lines
drivers/net/phy/phy.c +1
-0@@ -67,6 +67,7 @@ static void phy_link_down(struct phy_device *phydev) { phydev->phy_link_change (phydev, false);phy_led_trigger_change_speed (phydev); +WRITE_ONCE (phydev->link_down_events, phydev->link_down_events +1 );
I'm not sure the WRITE_ONCE adds much value. Many systems using PHYLIB are 32 bit, and i don't think WRITE_ONCE will make that 64 bit write atomic on 32 bit systems. And as Florian pointed out, you have bigger problems if you manged to overflow a u32 into a u64.
include/linux/phy.h struct phy_device { · line 727
Quoted 5 lines
include/linux/phy.h +1
-0@@ -723,6 +724,8 @@ struct phy_device {int pma_extable; +unsigned int link_down_events;
And here is unsigned int, not u64? Or u32? It would be good to be consistent.
include/uapi/linux/ethtool_netlink.h enum { · line 266
Quoted 8 lines
include/uapi/linux/ethtool_netlink.h +2
-0@@ -262,6 +262,8 @@ enum {ETHTOOL_A_LINKSTATE_SQI_MAX ,/* u32 */ ETHTOOL_A_LINKSTATE_EXT_STATE ,/* u8 */ ETHTOOL_A_LINKSTATE_EXT_SUBSTATE ,/* u8 */ +ETHTOOL_A_LINKSTATE_PAD , + ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT, /* u64 */
What is the PAD for?
Andrew
Jakub Kicinski
maintainerQuoted 11 lines · On Fri, 28 Oct 2022 14:18:11 +0200 Andrew Lunn wrote:
drivers/net/phy/phy.c static void phy_link_down(struct phy_device *phydev) · line 70
Quoted 5 lines
drivers/net/phy/phy.c +1
-0@@ -67,6 +67,7 @@ static void phy_link_down(struct phy_device *phydev) { phydev->phy_link_change (phydev, false);phy_led_trigger_change_speed (phydev); +WRITE_ONCE (phydev->link_down_events, phydev->link_down_events +1 );I'm not sure the
WRITE_ONCEadds much value. Many systems using PHYLIB are 32 bit, and i don't thinkWRITE_ONCEwill make that 64 bit write atomic on 32 bit systems. And as Florian pointed out, you have bigger problems if you manged to overflow a u32 into a u64.
Quoted 8 lines
include/linux/phy.h struct phy_device { · line 727
Quoted 5 lines
include/linux/phy.h +1
-0@@ -723,6 +724,8 @@ struct phy_device {int pma_extable; +unsigned int link_down_events;And here is unsigned int, not u64? Or u32? It would be good to be consistent.
I made it 32b on the phylib side so that WRITE_ONCE() was sufficient to ensure atomic writes.
Do you have a preference for it being 64b vs 32b at the uAPI level? I was leaning slightly towards making both 32b in v3..
Quoted 10 lines
include/uapi/linux/ethtool_netlink.h enum { · line 266
Quoted 8 lines
include/uapi/linux/ethtool_netlink.h +2
-0@@ -262,6 +262,8 @@ enum {ETHTOOL_A_LINKSTATE_SQI_MAX ,/* u32 */ ETHTOOL_A_LINKSTATE_EXT_STATE ,/* u8 */ ETHTOOL_A_LINKSTATE_EXT_SUBSTATE ,/* u8 */ +ETHTOOL_A_LINKSTATE_PAD , + ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT, /* u64 */What is the PAD for?
64b values have to be padded in netlink to ensure alignment.
Andrew Lunn
maintainerQuoted 2 lines
Do you have a preference for it being 64b vs 32b at the uAPI level? I was leaning slightly towards making both 32b in v3..
32 bit should be sufficient, and keeps everything simple.
Andrew