bnxt: add rx discards stats for oom and netpool
Jakub Kicinski
maintainerv3 cover bnxt: add rx discards stats for oom and netpool
Drivers should avoid silently dropping frames. This set adds two stats for previously unaccounted events to bnxt - packets dropped due to allocation failures and packets dropped during emergency ring polling.
Jakub Kicinski (3):
bnxt: reorder logic in bnxt_get_stats64()
bnxt: count packets discarded because of netpoll
bnxt: count discards due to memory allocation errors
drivers/net/ethernet/broadcom/bnxt/bnxt.c | 54 ++++++++++++++----- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 3 ++ .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 6 +++ 3 files changed, 50 insertions(+), 13 deletions(-)
Signature · 1 line
2.31.1
Jakub Kicinski
maintainerPATCH v3 1/3 bnxt: reorder logic in bnxt_get_stats64()
Saved ring stats and port stats are completely disjoint. We can reorder getting the old stats and collecting new stats from rings with reading port stats.
We can also use bnxt_add_prev_stats() instead of doing a struct assignment.
With that we can use the same code for closed and open device, next commits will add more stats at the end of bnxt_get_stats64().
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +7 -8
@@ -10673,14 +10673,8 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)* we check the BNXT_STATE_OPEN flag. */ smp_mb__after_atomic (); -if (!test_bit (BNXT_STATE_OPEN , &bp->state)) { -clear_bit (BNXT_STATE_READ_STATS , &bp->state); - *stats = bp->net_stats_prev; -return ; - } - -bnxt_get_ring_stats (bp, stats); -bnxt_add_prev_stats (bp, stats); +if (!test_bit (BNXT_STATE_OPEN , &bp->state)) + goto skip_current;if (bp->flags &BNXT_FLAG_PORT_STATS ) {u64 *rx = bp->port_stats.sw_stats;
@@ -10704,6 +10698,11 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)BNXT_GET_TX_PORT_STATS64 (tx, tx_fifo_underruns); stats->tx_errors =BNXT_GET_TX_PORT_STATS64 (tx, tx_err); } + +bnxt_get_ring_stats (bp, stats); +skip_current: +bnxt_add_prev_stats (bp, stats); +clear_bit (BNXT_STATE_READ_STATS , &bp->state); }
Signature · 1 line
2.31.1
Jakub Kicinski
maintainerPATCH v3 2/3 bnxt: count packets discarded because of netpoll
In reply to Jakub Kicinskibnxt may discard packets if Rx completions are consumed in an attempt to let netpoll make progress. It should be extremely rare in practice but nonetheless such events should be counted.
Since completion ring memory is allocated dynamically use a similar scheme to what is done for HW stats to save them.
Report the stats in rx_dropped and per-netdev ethtool counter. Chances that users care which ring dropped are very low.
--
v2: don't count ret == 0 case [Michael]
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 31 ++++++++++++++----- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 2 ++ .../net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 6 ++++ 3 files changed, 32 insertions(+), 7 deletions(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +24 -7
@@ -2003,6 +2003,7 @@ static int bnxt_force_rx_discard(struct bnxt *bp,struct rx_cmp *rxcmp;u16 cp_cons;u8 cmp_type; +int ret; cp_cons =RING_CMP (tmp_raw_cons); rxcmp = (struct rx_cmp *)
@@ -2031,7 +2032,10 @@ static int bnxt_force_rx_discard(struct bnxt *bp, tpa_end1->rx_tpa_end_cmp_errors_v2 |=cpu_to_le32 (RX_TPA_END_CMP_ERRORS ); } -return bnxt_rx_pkt (bp, cpr, raw_cons, event); + ret =bnxt_rx_pkt (bp, cpr, raw_cons, event); +if (ret && ret != -EBUSY ) + cpr->sw_stats.rx.rx_netpoll_discards +=1 ; +return ret; }u32 bnxt_fw_health_readl (struct bnxt *bp,int reg_idx)
@@ -10441,7 +10445,8 @@ static bool bnxt_drv_busy(struct bnxt *bp) }static void bnxt_get_ring_stats (struct bnxt *bp, -struct rtnl_link_stats64 *stats); +struct rtnl_link_stats64 *stats, +struct bnxt_sw_stats *bsw_stats);static void __bnxt_close_nic (struct bnxt *bp,bool irq_re_init,bool link_re_init)
@@ -10470,7 +10475,8 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,/* Save ring stats before shutdown */ if (bp->bnapi && irq_re_init) -bnxt_get_ring_stats (bp, &bp->net_stats_prev); +bnxt_get_ring_stats (bp, &bp->net_stats_prev, + &bp->sw_stats_prev);if (irq_re_init) {bnxt_free_irq (bp);bnxt_del_napi (bp);
@@ -10615,7 +10621,8 @@ static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) }static void bnxt_get_ring_stats (struct bnxt *bp, -struct rtnl_link_stats64 *stats) +struct rtnl_link_stats64 *stats, +struct bnxt_sw_stats *bsw_stats) {int i;
@@ -10646,11 +10653,15 @@ static void bnxt_get_ring_stats(struct bnxt *bp, stats->multicast +=BNXT_GET_RING_STATS64 (sw, rx_mcast_pkts); stats->tx_dropped +=BNXT_GET_RING_STATS64 (sw, tx_error_pkts); + + bsw_stats->rx.rx_netpoll_discards += + cpr->sw_stats.rx.rx_netpoll_discards; } }static void bnxt_add_prev_stats (struct bnxt *bp, -struct rtnl_link_stats64 *stats) +struct rtnl_link_stats64 *stats, +struct bnxt_sw_stats *bsw_stats) {struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
@@ -10661,11 +10672,15 @@ static void bnxt_add_prev_stats(struct bnxt *bp, stats->rx_missed_errors += prev_stats->rx_missed_errors; stats->multicast += prev_stats->multicast; stats->tx_dropped += prev_stats->tx_dropped; + + bsw_stats->rx.rx_netpoll_discards += + bp->sw_stats_prev.rx.rx_netpoll_discards; }static void bnxt_get_stats64 (struct net_device *dev,struct rtnl_link_stats64 *stats) { +struct bnxt_sw_stats bsw_stats = {};struct bnxt *bp =netdev_priv (dev);set_bit (BNXT_STATE_READ_STATS , &bp->state);
@@ -10699,9 +10714,11 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) stats->tx_errors =BNXT_GET_TX_PORT_STATS64 (tx, tx_err); } -bnxt_get_ring_stats (bp, stats); +bnxt_get_ring_stats (bp, stats, &bsw_stats); skip_current: -bnxt_add_prev_stats (bp, stats); +bnxt_add_prev_stats (bp, stats, &bsw_stats); + + stats->rx_dropped += bsw_stats.rx.rx_netpoll_discards;clear_bit (BNXT_STATE_READ_STATS , &bp->state); }
drivers/net/ethernet/broadcom/bnxt/bnxt.h +2 -0
@@ -939,6 +939,7 @@ struct bnxt_rx_sw_stats {u64 rx_l4_csum_errors;u64 rx_resets;u64 rx_buf_errors; +u64 rx_netpoll_discards; };struct bnxt_cmn_sw_stats {
@@ -1917,6 +1918,7 @@ struct bnxt {dma_addr_t hwrm_cmd_kong_resp_dma_addr;struct rtnl_link_stats64 net_stats_prev; +struct bnxt_sw_stats sw_stats_prev;struct bnxt_stats_mem port_stats;struct bnxt_stats_mem rx_port_stats_ext;struct bnxt_stats_mem tx_port_stats_ext;
drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +6 -0
@@ -307,6 +307,7 @@ static const char * const bnxt_cmn_sw_stats_str[] = {enum {RX_TOTAL_DISCARDS ,TX_TOTAL_DISCARDS , +RX_NETPOLL_DISCARDS , }; static struct {
@@ -315,6 +316,7 @@ static struct { } bnxt_sw_func_stats[] = { {0, "rx_total_discard_pkts"}, {0, "tx_total_discard_pkts"}, + {0, "rx_netpoll_discards"}, }; #define NUM_RING_RX_SW_STATS ARRAY_SIZE(bnxt_rx_sw_stats_str)
@@ -561,6 +563,8 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,for (i =0 ; i <BNXT_NUM_SW_FUNC_STATS ; i++) bnxt_sw_func_stats[i].counter =0 ; + bnxt_sw_func_stats[RX_NETPOLL_DISCARDS ].counter = + bp->sw_stats_prev.rx.rx_netpoll_discards; tpa_stats =bnxt_get_num_tpa_ring_stats (bp);for (i =0 ; i < bp->cp_nr_rings; i++) {
@@ -603,6 +607,8 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,BNXT_GET_RING_STATS64 (sw_stats, rx_discard_pkts); bnxt_sw_func_stats[TX_TOTAL_DISCARDS ].counter +=BNXT_GET_RING_STATS64 (sw_stats, tx_discard_pkts); + bnxt_sw_func_stats[RX_NETPOLL_DISCARDS ].counter += + cpr->sw_stats.rx.rx_netpoll_discards; }for (i =0 ; i <BNXT_NUM_SW_FUNC_STATS ; i++, j++)
Signature · 1 line
2.31.1
Michael Chan
maintainerRe: [PATCH net-next v3 2/3] bnxt: count packets discarded because of netpoll
drivers/net/ethernet/broadcom/bnxt/bnxt.c static void bnxt_get_ring_stats(struct bnxt *bp, · line 10658
Quoted 7 lines · Jakub Kicinski
drivers/net/ethernet/broadcom/bnxt/bnxt.c +3
-0@@ -10646,11 +10653,15 @@ static void bnxt_get_ring_stats(struct bnxt *bp, stats->multicast +=BNXT_GET_RING_STATS64 (sw, rx_mcast_pkts); stats->tx_dropped +=BNXT_GET_RING_STATS64 (sw, tx_error_pkts); + + bsw_stats->rx.rx_netpoll_discards += + cpr->sw_stats.rx.rx_netpoll_discards;
Can we just add these rx_netpoll_discards counters directly to stats->rx_dropped? It looks simpler if we do it that way, right?
Quoted 10 lines
drivers/net/ethernet/broadcom/bnxt/bnxt.c +2
-1} }static void bnxt_add_prev_stats (struct bnxt *bp, -struct rtnl_link_stats64 *stats) +struct rtnl_link_stats64 *stats, +struct bnxt_sw_stats *bsw_stats) {struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
Jakub Kicinski
maintainerRe: [PATCH net-next v3 2/3] bnxt: count packets discarded because of netpoll
Quoted 11 lines · On Thu, 26 Aug 2021 11:43:58 -0700 Michael Chan wrote:
drivers/net/ethernet/broadcom/bnxt/bnxt.c static void bnxt_get_ring_stats(struct bnxt *bp, · line 10658
Quoted 7 lines · Jakub Kicinski
drivers/net/ethernet/broadcom/bnxt/bnxt.c +3
-0@@ -10646,11 +10653,15 @@ static void bnxt_get_ring_stats(struct bnxt *bp, stats->multicast +=BNXT_GET_RING_STATS64 (sw, rx_mcast_pkts); stats->tx_dropped +=BNXT_GET_RING_STATS64 (sw, tx_error_pkts); + + bsw_stats->rx.rx_netpoll_discards += + cpr->sw_stats.rx.rx_netpoll_discards;Can we just add these
rx_netpoll_discardscounters directly tostats->rx_dropped? It looks simpler if we do it that way, right?
To make sure - are you saying that instead of adding
struct bnxt_sw_stats sw_stats_prev;
we should accumulate in net_stats_prev->rx_dropped, and have the ethtool counter only report the discards since last down/up?
Or to use the atomic counter on the netdev and never report in ethtool (since after patch 3 rx_dropped is a mix of reasons)?
Michael Chan
maintainerRe: [PATCH net-next v3 2/3] bnxt: count packets discarded because of netpoll
Quoted 16 lines · Jakub Kicinski
Quoted 11 lines · On Thu, 26 Aug 2021 11:43:58 -0700 Michael Chan wrote:
drivers/net/ethernet/broadcom/bnxt/bnxt.c static void bnxt_get_ring_stats(struct bnxt *bp, · line 10658
Quoted 7 lines · Jakub Kicinski
drivers/net/ethernet/broadcom/bnxt/bnxt.c +3
-0@@ -10646,11 +10653,15 @@ static void bnxt_get_ring_stats(struct bnxt *bp, stats->multicast +=BNXT_GET_RING_STATS64 (sw, rx_mcast_pkts); stats->tx_dropped +=BNXT_GET_RING_STATS64 (sw, tx_error_pkts); + + bsw_stats->rx.rx_netpoll_discards += + cpr->sw_stats.rx.rx_netpoll_discards;Can we just add these
rx_netpoll_discardscounters directly tostats->rx_dropped? It looks simpler if we do it that way, right?To make sure - are you saying that instead of adding
Quoted 7 lines
struct bnxt_sw_stats sw_stats_prev;we should accumulate in
net_stats_prev->rx_dropped, and have the ethtool counter only report the discards since last down/up?Or to use the atomic counter on the netdev and never report in ethtool (since after patch 3
rx_droppedis a mix of reasons)?
OK. I've reviewed the patch again and you need to keep the previous netpoll discard counter so that you can report the total current and previous netpoll discard counter under ethtool -S.
My suggestion would lump the previous netpoll discard counter into the previous rx_dropped counter and you can only report the current netpoll discard counter under ethtool -S. But note that all the ring related counters we currently report are current counters and do not include old counters before the last reset.
Jakub Kicinski
maintainerRe: [PATCH net-next v3 2/3] bnxt: count packets discarded because of netpoll
Quoted 19 lines · On Thu, 26 Aug 2021 14:17:45 -0700 Michael Chan wrote:
Quoted 6 lines · Jakub Kicinski
Quoted 2 lines · On Thu, 26 Aug 2021 11:43:58 -0700 Michael Chan wrote:
Can we just add these
rx_netpoll_discardscounters directly tostats->rx_dropped? It looks simpler if we do it that way, right?To make sure - are you saying that instead of adding
Quoted 7 lines
struct bnxt_sw_stats sw_stats_prev;we should accumulate in
net_stats_prev->rx_dropped, and have the ethtool counter only report the discards since last down/up?Or to use the atomic counter on the netdev and never report in ethtool (since after patch 3
rx_droppedis a mix of reasons)?OK. I've reviewed the patch again and you need to keep the previous netpoll discard counter so that you can report the total current and previous netpoll discard counter under ethtool -S.
Quoted 5 lines
My suggestion would lump the previous netpoll discard counter into the previous
rx_droppedcounter and you can only report the current netpoll discard counter under ethtool -S. But note that all the ring related counters we currently report are current counters and do not include old counters before the last reset.
Oh, [rt]x_total_discard_pkts are also just a sum of current counters? I missed that. In that case if netpoll discards reset it's not a big deal, I'll respin the patch tomorrow. Let me also rename from rx_netpoll_discards to rx_total_netpoll_discards, adding the "total_" will hopefully signal the similarity of semantics?
Jakub Kicinski
maintainerPATCH v3 3/3 bnxt: count discards due to memory allocation errors
In reply to Jakub KicinskiCount packets dropped due to buffer or skb allocation errors. Report as part of rx_dropped.
v2: drop the ethtool -S entry [Vladimir]
--- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 14 +++++++++++++- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 1 + 2 files changed, 14 insertions(+), 1 deletion(-)
drivers/net/ethernet/broadcom/bnxt/bnxt.c +13 -1
@@ -1651,6 +1651,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, skb =bnxt_copy_skb (bnapi, data_ptr, len, mapping);if (!skb) {bnxt_abort_tpa (cpr, idx, agg_bufs); + cpr->sw_stats.rx.rx_oom_discards +=1 ;return NULL ; } }else {
@@ -1660,6 +1661,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, new_data =__bnxt_alloc_rx_data (bp, &new_mapping,GFP_ATOMIC );if (!new_data) {bnxt_abort_tpa (cpr, idx, agg_bufs); + cpr->sw_stats.rx.rx_oom_discards +=1 ;return NULL ; }
@@ -1675,6 +1677,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,if (!skb) {kfree (data);bnxt_abort_tpa (cpr, idx, agg_bufs); + cpr->sw_stats.rx.rx_oom_discards +=1 ;return NULL ; }skb_reserve (skb, bp->rx_offset);
@@ -1685,6 +1688,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, skb =bnxt_rx_pages (bp, cpr, skb, idx, agg_bufs, true);if (!skb) {/* Page reuse already handled by bnxt_rx_pages(). */ + cpr->sw_stats.rx.rx_oom_discards +=1 ;return NULL ; } }
@@ -1888,6 +1892,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,if (agg_bufs)bnxt_reuse_rx_agg_bufs (cpr, cp_cons,0 , agg_bufs, false); + cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ; goto next_rx; }
@@ -1901,6 +1906,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, skb = bp->rx_skb_func (bp, rxr, cons, data, data_ptr, dma_addr, payload | len);if (!skb) { + cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ; goto next_rx; }
@@ -1909,6 +1915,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,if (agg_bufs) { skb =bnxt_rx_pages (bp, cpr, skb, cp_cons, agg_bufs, false);if (!skb) { + cpr->sw_stats.rx.rx_oom_discards +=1 ; rc = -ENOMEM ; goto next_rx; }
@@ -10656,6 +10663,8 @@ static void bnxt_get_ring_stats(struct bnxt *bp, bsw_stats->rx.rx_netpoll_discards += cpr->sw_stats.rx.rx_netpoll_discards; + bsw_stats->rx.rx_oom_discards += + cpr->sw_stats.rx.rx_oom_discards; } }
@@ -10675,6 +10684,7 @@ static void bnxt_add_prev_stats(struct bnxt *bp, bsw_stats->rx.rx_netpoll_discards += bp->sw_stats_prev.rx.rx_netpoll_discards; + bsw_stats->rx.rx_oom_discards += bp->sw_stats_prev.rx.rx_oom_discards; } static void
@@ -10718,7 +10728,9 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) skip_current:bnxt_add_prev_stats (bp, stats, &bsw_stats); - stats->rx_dropped += bsw_stats.rx.rx_netpoll_discards; + stats->rx_dropped += + bsw_stats.rx.rx_netpoll_discards + + bsw_stats.rx.rx_oom_discards;clear_bit (BNXT_STATE_READ_STATS , &bp->state); }
drivers/net/ethernet/broadcom/bnxt/bnxt.h +1 -0
@@ -939,6 +939,7 @@ struct bnxt_rx_sw_stats {u64 rx_l4_csum_errors;u64 rx_resets;u64 rx_buf_errors; +u64 rx_oom_discards;u64 rx_netpoll_discards; };
Signature · 1 line
2.31.1