net: adopt u64_stats_t type
Subsystems (11)
v2 cover net: adopt u64_stats_t type
From: Eric Dumazet <[email protected]>
While KCSAN has not raised any reports yet, we should address the potential load/store tearing problem happening with per cpu stats.
This series is not exhaustive, but hopefully a step in the right direction.
v2: change title in wireguard patch, and fix cover letter title.
Eric Dumazet (9):
vlan: adopt u64_stats_t
ipvlan: adopt u64_stats_t
sit: use dev_sw_netstats_rx_add()
ip6_tunnel: use dev_sw_netstats_rx_add()
wireguard: receive: use dev_sw_netstats_rx_add()
net: adopt u64_stats_t in struct pcpu_sw_netstats
devlink: adopt u64_stats_t
drop_monitor: adopt u64_stats_t
team: adopt u64_stats_t
drivers/net/ipvlan/ipvlan.h | 10 ++++----- drivers/net/ipvlan/ipvlan_core.c | 6 +++--- drivers/net/ipvlan/ipvlan_main.c | 18 ++++++++-------- drivers/net/macsec.c | 8 +++---- drivers/net/macvlan.c | 18 ++++++++-------- drivers/net/team/team.c | 26 +++++++++++------------ drivers/net/usb/usbnet.c | 8 +++---- drivers/net/vxlan/vxlan_core.c | 8 +++---- drivers/net/wireguard/receive.c | 9 +------- include/linux/if_macvlan.h | 6 +++--- include/linux/if_team.h | 10 ++++----- include/linux/if_vlan.h | 10 ++++----- include/linux/netdevice.h | 16 +++++++------- include/net/ip_tunnels.h | 4 ++-- net/8021q/vlan_core.c | 6 +++--- net/8021q/vlan_dev.c | 18 ++++++++-------- net/bridge/br_netlink.c | 8 +++---- net/bridge/br_vlan.c | 36 ++++++++++++++++++-------------- net/core/dev.c | 18 ++++++++-------- net/core/devlink.c | 28 ++++++++++++++----------- net/core/drop_monitor.c | 18 ++++++++-------- net/dsa/slave.c | 8 +++---- net/ipv6/ip6_tunnel.c | 7 +------ net/ipv6/sit.c | 8 +------ 24 files changed, 151 insertions(+), 161 deletions(-)
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 1/9 vlan: adopt u64_stats_t
From: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
Add READ_ONCE() when reading rx_errors & tx_dropped.
--- drivers/net/macvlan.c | 18 +++++++++--------- include/linux/if_macvlan.h | 6 +++--- include/linux/if_vlan.h | 10 +++++----- net/8021q/vlan_core.c | 6 +++--- net/8021q/vlan_dev.c | 18 +++++++++--------- 5 files changed, 29 insertions(+), 29 deletions(-)
drivers/net/macvlan.c +9 -9
@@ -575,8 +575,8 @@ static netdev_tx_t macvlan_start_xmit(struct sk_buff *skb, pcpu_stats =this_cpu_ptr (vlan->pcpu_stats);u64_stats_update_begin (&pcpu_stats->syncp); - pcpu_stats->tx_packets++; - pcpu_stats->tx_bytes += len; +u64_stats_inc (&pcpu_stats->tx_packets); +u64_stats_add (&pcpu_stats->tx_bytes, len);u64_stats_update_end (&pcpu_stats->syncp); }else {this_cpu_inc (vlan->pcpu_stats->tx_dropped);
@@ -949,11 +949,11 @@ static void macvlan_dev_get_stats64(struct net_device *dev, p =per_cpu_ptr (vlan->pcpu_stats, i);do { start =u64_stats_fetch_begin_irq (&p->syncp); - rx_packets = p->rx_packets; - rx_bytes = p->rx_bytes; - rx_multicast = p->rx_multicast; - tx_packets = p->tx_packets; - tx_bytes = p->tx_bytes; + rx_packets =u64_stats_read (&p->rx_packets); + rx_bytes =u64_stats_read (&p->rx_bytes); + rx_multicast =u64_stats_read (&p->rx_multicast); + tx_packets =u64_stats_read (&p->tx_packets); + tx_bytes =u64_stats_read (&p->tx_bytes); }while (u64_stats_fetch_retry_irq (&p->syncp, start)); stats->rx_packets += rx_packets;
@@ -964,8 +964,8 @@ static void macvlan_dev_get_stats64(struct net_device *dev,/* rx_errors & tx_dropped are u32, updated * without syncp protection. */ - rx_errors += p->rx_errors; - tx_dropped += p->tx_dropped; + rx_errors +=READ_ONCE (p->rx_errors); + tx_dropped +=READ_ONCE (p->tx_dropped); } stats->rx_errors = rx_errors; stats->rx_dropped = rx_errors;
include/linux/if_macvlan.h +3 -3
@@ -46,10 +46,10 @@ static inline void macvlan_count_rx(const struct macvlan_dev *vlan, pcpu_stats =get_cpu_ptr (vlan->pcpu_stats);u64_stats_update_begin (&pcpu_stats->syncp); - pcpu_stats->rx_packets++; - pcpu_stats->rx_bytes += len; +u64_stats_inc (&pcpu_stats->rx_packets); +u64_stats_add (&pcpu_stats->rx_bytes, len);if (multicast) - pcpu_stats->rx_multicast++; +u64_stats_inc (&pcpu_stats->rx_multicast);u64_stats_update_end (&pcpu_stats->syncp);put_cpu_ptr (vlan->pcpu_stats); }else {
include/linux/if_vlan.h +5 -5
@@ -118,11 +118,11 @@ static inline void vlan_drop_rx_stag_filter_info(struct net_device *dev)* @tx_dropped: number of tx drops */ struct vlan_pcpu_stats { -u64 rx_packets; -u64 rx_bytes; -u64 rx_multicast; -u64 tx_packets; -u64 tx_bytes; +u64_stats_t rx_packets; +u64_stats_t rx_bytes; +u64_stats_t rx_multicast; +u64_stats_t tx_packets; +u64_stats_t tx_bytes;struct u64_stats_sync syncp;u32 rx_errors;u32 tx_dropped;
net/8021q/vlan_core.c +3 -3
@@ -63,10 +63,10 @@ bool vlan_do_receive(struct sk_buff **skbp) rx_stats =this_cpu_ptr (vlan_dev_priv (vlan_dev)->vlan_pcpu_stats);u64_stats_update_begin (&rx_stats->syncp); - rx_stats->rx_packets++; - rx_stats->rx_bytes += skb->len; +u64_stats_inc (&rx_stats->rx_packets); +u64_stats_add (&rx_stats->rx_bytes, skb->len);if (skb->pkt_type ==PACKET_MULTICAST ) - rx_stats->rx_multicast++; +u64_stats_inc (&rx_stats->rx_multicast);u64_stats_update_end (&rx_stats->syncp);return true;
net/8021q/vlan_dev.c +9 -9
@@ -128,8 +128,8 @@ static netdev_tx_t vlan_dev_hard_start_xmit(struct sk_buff *skb, stats =this_cpu_ptr (vlan->vlan_pcpu_stats);u64_stats_update_begin (&stats->syncp); - stats->tx_packets++; - stats->tx_bytes += len; +u64_stats_inc (&stats->tx_packets); +u64_stats_add (&stats->tx_bytes, len);u64_stats_update_end (&stats->syncp); }else {this_cpu_inc (vlan->vlan_pcpu_stats->tx_dropped);
@@ -713,11 +713,11 @@ static void vlan_dev_get_stats64(struct net_device *dev, p =per_cpu_ptr (vlan_dev_priv (dev)->vlan_pcpu_stats, i);do { start =u64_stats_fetch_begin_irq (&p->syncp); - rxpackets = p->rx_packets; - rxbytes = p->rx_bytes; - rxmulticast = p->rx_multicast; - txpackets = p->tx_packets; - txbytes = p->tx_bytes; + rxpackets =u64_stats_read (&p->rx_packets); + rxbytes =u64_stats_read (&p->rx_bytes); + rxmulticast =u64_stats_read (&p->rx_multicast); + txpackets =u64_stats_read (&p->tx_packets); + txbytes =u64_stats_read (&p->tx_bytes); }while (u64_stats_fetch_retry_irq (&p->syncp, start)); stats->rx_packets += rxpackets;
@@ -726,8 +726,8 @@ static void vlan_dev_get_stats64(struct net_device *dev, stats->tx_packets += txpackets; stats->tx_bytes += txbytes;/* rx_errors & tx_dropped are u32 */ - rx_errors += p->rx_errors; - tx_dropped += p->tx_dropped; + rx_errors +=READ_ONCE (p->rx_errors); + tx_dropped +=READ_ONCE (p->tx_dropped); } stats->rx_errors = rx_errors; stats->tx_dropped = tx_dropped;
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 2/9 ipvlan: adopt u64_stats_t
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
Add READ_ONCE() when reading rx_errs & tx_drps.
--- drivers/net/ipvlan/ipvlan.h | 10 +++++----- drivers/net/ipvlan/ipvlan_core.c | 6 +++--- drivers/net/ipvlan/ipvlan_main.c | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-)
drivers/net/ipvlan/ipvlan.h +5 -5
@@ -47,11 +47,11 @@ typedef enum { } ipvl_hdr_type;struct ipvl_pcpu_stats { -u64 rx_pkts; -u64 rx_bytes; -u64 rx_mcast; -u64 tx_pkts; -u64 tx_bytes; +u64_stats_t rx_pkts; +u64_stats_t rx_bytes; +u64_stats_t rx_mcast; +u64_stats_t tx_pkts; +u64_stats_t tx_bytes;struct u64_stats_sync syncp;u32 rx_errs;u32 tx_drps;
drivers/net/ipvlan/ipvlan_core.c +3 -3
@@ -19,10 +19,10 @@ void ipvlan_count_rx(const struct ipvl_dev *ipvlan, pcptr =this_cpu_ptr (ipvlan->pcpu_stats);u64_stats_update_begin (&pcptr->syncp); - pcptr->rx_pkts++; - pcptr->rx_bytes += len; +u64_stats_inc (&pcptr->rx_pkts); +u64_stats_add (&pcptr->rx_bytes, len);if (mcast) - pcptr->rx_mcast++; +u64_stats_inc (&pcptr->rx_mcast);u64_stats_update_end (&pcptr->syncp); }else {this_cpu_inc (ipvlan->pcpu_stats->rx_errs);
drivers/net/ipvlan/ipvlan_main.c +9 -9
@@ -224,8 +224,8 @@ static netdev_tx_t ipvlan_start_xmit(struct sk_buff *skb, pcptr =this_cpu_ptr (ipvlan->pcpu_stats);u64_stats_update_begin (&pcptr->syncp); - pcptr->tx_pkts++; - pcptr->tx_bytes += skblen; +u64_stats_inc (&pcptr->tx_pkts); +u64_stats_add (&pcptr->tx_bytes, skblen);u64_stats_update_end (&pcptr->syncp); }else {this_cpu_inc (ipvlan->pcpu_stats->tx_drps);
@@ -300,11 +300,11 @@ static void ipvlan_get_stats64(struct net_device *dev, pcptr =per_cpu_ptr (ipvlan->pcpu_stats, idx);do { strt=u64_stats_fetch_begin_irq (&pcptr->syncp); - rx_pkts = pcptr->rx_pkts; - rx_bytes = pcptr->rx_bytes; - rx_mcast = pcptr->rx_mcast; - tx_pkts = pcptr->tx_pkts; - tx_bytes = pcptr->tx_bytes; + rx_pkts =u64_stats_read (&pcptr->rx_pkts); + rx_bytes =u64_stats_read (&pcptr->rx_bytes); + rx_mcast =u64_stats_read (&pcptr->rx_mcast); + tx_pkts =u64_stats_read (&pcptr->tx_pkts); + tx_bytes =u64_stats_read (&pcptr->tx_bytes); }while (u64_stats_fetch_retry_irq (&pcptr->syncp, strt));
@@ -315,8 +315,8 @@ static void ipvlan_get_stats64(struct net_device *dev, s->tx_bytes += tx_bytes;/* u32 values are updated without syncp protection. */ - rx_errs += pcptr->rx_errs; - tx_drps += pcptr->tx_drps; + rx_errs +=READ_ONCE (pcptr->rx_errs); + tx_drps +=READ_ONCE (pcptr->tx_drps); } s->rx_errors = rx_errs; s->rx_dropped = rx_errs;
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 3/9 sit: use dev_sw_netstats_rx_add()
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
We have a convenient helper, let's use it.
--- net/ipv6/sit.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
net/ipv6/sit.c +1 -7
@@ -686,8 +686,6 @@ static int ipip6_rcv(struct sk_buff *skb) tunnel =ipip6_tunnel_lookup (dev_net (skb->dev), skb->dev, iph->saddr, iph->daddr, sifindex);if (tunnel) { -struct pcpu_sw_netstats *tstats; -if (tunnel->parms.iph.protocol !=IPPROTO_IPV6 && tunnel->parms.iph.protocol !=0 ) goto out;
@@ -724,11 +722,7 @@ static int ipip6_rcv(struct sk_buff *skb) } } - tstats =this_cpu_ptr (tunnel->dev->tstats); -u64_stats_update_begin (&tstats->syncp); - tstats->rx_packets++; - tstats->rx_bytes += skb->len; -u64_stats_update_end (&tstats->syncp); +dev_sw_netstats_rx_add (tunnel->dev, skb->len);netif_rx (skb);
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 4/9 ip6_tunnel: use dev_sw_netstats_rx_add()
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
We have a convenient helper, let's use it.
--- net/ipv6/ip6_tunnel.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-)
net/ipv6/ip6_tunnel.c +1 -6
@@ -796,7 +796,6 @@ static int __ip6_tnl_rcv(struct ip6_tnl *tunnel, struct sk_buff *skb, struct sk_buff *skb), bool log_ecn_err) { -struct pcpu_sw_netstats *tstats;const struct ipv6hdr *ipv6h =ipv6_hdr (skb);int err;
@@ -856,11 +855,7 @@ static int __ip6_tnl_rcv(struct ip6_tnl *tunnel, struct sk_buff *skb, } } - tstats =this_cpu_ptr (tunnel->dev->tstats); -u64_stats_update_begin (&tstats->syncp); - tstats->rx_packets++; - tstats->rx_bytes += skb->len; -u64_stats_update_end (&tstats->syncp); +dev_sw_netstats_rx_add (tunnel->dev, skb->len);skb_scrub_packet (skb, !net_eq (tunnel->net,dev_net (tunnel->dev)));
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 5/9 wireguard: receive: use dev_sw_netstats_rx_add()
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
We have a convenient helper, let's use it.
This will make the following patch easier to review and smaller.
--- drivers/net/wireguard/receive.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-)
drivers/net/wireguard/receive.c +1 -8
@@ -19,15 +19,8 @@/* Must be called with bh disabled. */ static void update_rx_stats (struct wg_peer *peer,size_t len) { -struct pcpu_sw_netstats *tstats = -get_cpu_ptr (peer->device->dev->tstats); - -u64_stats_update_begin (&tstats->syncp); - ++tstats->rx_packets; - tstats->rx_bytes += len; +dev_sw_netstats_rx_add (peer->device->dev, len); peer->rx_bytes += len; -u64_stats_update_end (&tstats->syncp); -put_cpu_ptr (tstats); }#define SKB_TYPE_LE32 (skb) (((struct message_header *)(skb)->data)->type)
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 6/9 net: adopt u64_stats_t in struct pcpu_sw_netstats
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
--- drivers/net/macsec.c | 8 ++++---- drivers/net/usb/usbnet.c | 8 ++++---- drivers/net/vxlan/vxlan_core.c | 8 ++++---- include/linux/netdevice.h | 16 +++++++-------- include/net/ip_tunnels.h | 4 ++-- net/bridge/br_netlink.c | 8 ++++---- net/bridge/br_vlan.c | 36 +++++++++++++++++++--------------- net/core/dev.c | 18 ++++++++--------- net/dsa/slave.c | 8 ++++---- 9 files changed, 59 insertions(+), 55 deletions(-)
drivers/net/macsec.c +4 -4
@@ -523,8 +523,8 @@ static void count_tx(struct net_device *dev, int ret, int len)struct pcpu_sw_netstats *stats =this_cpu_ptr (dev->tstats);u64_stats_update_begin (&stats->syncp); - stats->tx_packets++; - stats->tx_bytes += len; +u64_stats_inc (&stats->tx_packets); +u64_stats_add (&stats->tx_bytes, len);u64_stats_update_end (&stats->syncp); } }
@@ -825,8 +825,8 @@ static void count_rx(struct net_device *dev, int len)struct pcpu_sw_netstats *stats =this_cpu_ptr (dev->tstats);u64_stats_update_begin (&stats->syncp); - stats->rx_packets++; - stats->rx_bytes += len; +u64_stats_inc (&stats->rx_packets); +u64_stats_add (&stats->rx_bytes, len);u64_stats_update_end (&stats->syncp); }
drivers/net/usb/usbnet.c +4 -4
@@ -337,8 +337,8 @@ void usbnet_skb_return (struct usbnet *dev, struct sk_buff *skb) skb->protocol =eth_type_trans (skb, dev->net); flags =u64_stats_update_begin_irqsave (&stats64->syncp); - stats64->rx_packets++; - stats64->rx_bytes += skb->len; +u64_stats_inc (&stats64->rx_packets); +u64_stats_add (&stats64->rx_bytes, skb->len);u64_stats_update_end_irqrestore (&stats64->syncp, flags); netif_dbg(dev, rx_status, dev->net, "< rx, len %zu, type 0x%x\n",
@@ -1258,8 +1258,8 @@ static void tx_complete (struct urb *urb)unsigned long flags; flags =u64_stats_update_begin_irqsave (&stats64->syncp); - stats64->tx_packets += entry->packets; - stats64->tx_bytes += entry->length; +u64_stats_add (&stats64->tx_packets, entry->packets); +u64_stats_add (&stats64->tx_bytes, entry->length);u64_stats_update_end_irqrestore (&stats64->syncp, flags); }else { dev->net->stats.tx_errors++;
drivers/net/vxlan/vxlan_core.c +4 -4
@@ -2385,15 +2385,15 @@ static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,vxlan_snoop (dev, &loopback,eth_hdr (skb)->h_source,0 , vni);u64_stats_update_begin (&tx_stats->syncp); - tx_stats->tx_packets++; - tx_stats->tx_bytes += len; +u64_stats_inc (&tx_stats->tx_packets); +u64_stats_add (&tx_stats->tx_bytes, len);u64_stats_update_end (&tx_stats->syncp);vxlan_vnifilter_count (src_vxlan, vni,NULL ,VXLAN_VNI_STATS_TX , len);if (__netif_rx (skb) ==NET_RX_SUCCESS ) {u64_stats_update_begin (&rx_stats->syncp); - rx_stats->rx_packets++; - rx_stats->rx_bytes += len; +u64_stats_inc (&rx_stats->rx_packets); +u64_stats_add (&rx_stats->rx_bytes, len);u64_stats_update_end (&rx_stats->syncp);vxlan_vnifilter_count (dst_vxlan, vni,NULL ,VXLAN_VNI_STATS_RX , len);
include/linux/netdevice.h +8 -8
@@ -2636,10 +2636,10 @@ struct packet_offload {/* often modified stats are per-CPU, other are shared (netdev->stats) */ struct pcpu_sw_netstats { -u64 rx_packets; -u64 rx_bytes; -u64 tx_packets; -u64 tx_bytes; +u64_stats_t rx_packets; +u64_stats_t rx_bytes; +u64_stats_t tx_packets; +u64_stats_t tx_bytes;struct u64_stats_sync syncp; }__aligned(4 * sizeof(u64)) ;
@@ -2656,8 +2656,8 @@ static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int lstruct pcpu_sw_netstats *tstats =this_cpu_ptr (dev->tstats);u64_stats_update_begin (&tstats->syncp); - tstats->rx_bytes += len; - tstats->rx_packets++; +u64_stats_add (&tstats->rx_bytes, len); +u64_stats_inc (&tstats->rx_packets);u64_stats_update_end (&tstats->syncp); }
@@ -2668,8 +2668,8 @@ static inline void dev_sw_netstats_tx_add(struct net_device *dev,struct pcpu_sw_netstats *tstats =this_cpu_ptr (dev->tstats);u64_stats_update_begin (&tstats->syncp); - tstats->tx_bytes += len; - tstats->tx_packets += packets; +u64_stats_add (&tstats->tx_bytes, len); +u64_stats_add (&tstats->tx_packets, packets);u64_stats_update_end (&tstats->syncp); }
include/net/ip_tunnels.h +2 -2
@@ -456,8 +456,8 @@ static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len)struct pcpu_sw_netstats *tstats =get_cpu_ptr (dev->tstats);u64_stats_update_begin (&tstats->syncp); - tstats->tx_bytes += pkt_len; - tstats->tx_packets++; +u64_stats_add (&tstats->tx_bytes, pkt_len); +u64_stats_inc (&tstats->tx_packets);u64_stats_update_end (&tstats->syncp);put_cpu_ptr (tstats); }else {
net/bridge/br_netlink.c +4 -4
@@ -1770,10 +1770,10 @@ static int br_fill_linkxstats(struct sk_buff *skb,if (v->vid == pvid) vxi.flags |=BRIDGE_VLAN_INFO_PVID ;br_vlan_get_stats (v, &stats); - vxi.rx_bytes = stats.rx_bytes; - vxi.rx_packets = stats.rx_packets; - vxi.tx_bytes = stats.tx_bytes; - vxi.tx_packets = stats.tx_packets; + vxi.rx_bytes =u64_stats_read (&stats.rx_bytes); + vxi.rx_packets =u64_stats_read (&stats.rx_packets); + vxi.tx_bytes =u64_stats_read (&stats.tx_bytes); + vxi.tx_packets =u64_stats_read (&stats.tx_packets);if (nla_put (skb,BRIDGE_XSTATS_VLAN ,sizeof (vxi), &vxi)) goto nla_put_failure;
net/bridge/br_vlan.c +20 -16
@@ -505,8 +505,8 @@ struct sk_buff *br_handle_vlan(struct net_bridge *br,if (br_opt_get (br,BROPT_VLAN_STATS_ENABLED )) { stats =this_cpu_ptr (v->stats);u64_stats_update_begin (&stats->syncp); - stats->tx_bytes += skb->len; - stats->tx_packets++; +u64_stats_add (&stats->tx_bytes, skb->len); +u64_stats_inc (&stats->tx_packets);u64_stats_update_end (&stats->syncp); }
@@ -624,8 +624,8 @@ static bool __allowed_ingress(const struct net_bridge *br,if (br_opt_get (br,BROPT_VLAN_STATS_ENABLED )) { stats =this_cpu_ptr (v->stats);u64_stats_update_begin (&stats->syncp); - stats->rx_bytes += skb->len; - stats->rx_packets++; +u64_stats_add (&stats->rx_bytes, skb->len); +u64_stats_inc (&stats->rx_packets);u64_stats_update_end (&stats->syncp); }
@@ -1379,16 +1379,16 @@ void br_vlan_get_stats(const struct net_bridge_vlan *v, cpu_stats =per_cpu_ptr (v->stats, i);do { start =u64_stats_fetch_begin_irq (&cpu_stats->syncp); - rxpackets = cpu_stats->rx_packets; - rxbytes = cpu_stats->rx_bytes; - txbytes = cpu_stats->tx_bytes; - txpackets = cpu_stats->tx_packets; + rxpackets =u64_stats_read (&cpu_stats->rx_packets); + rxbytes =u64_stats_read (&cpu_stats->rx_bytes); + txbytes =u64_stats_read (&cpu_stats->tx_bytes); + txpackets =u64_stats_read (&cpu_stats->tx_packets); }while (u64_stats_fetch_retry_irq (&cpu_stats->syncp, start)); - stats->rx_packets += rxpackets; - stats->rx_bytes += rxbytes; - stats->tx_bytes += txbytes; - stats->tx_packets += txpackets; +u64_stats_add (&stats->rx_packets, rxpackets); +u64_stats_add (&stats->rx_bytes, rxbytes); +u64_stats_add (&stats->tx_bytes, txbytes); +u64_stats_add (&stats->tx_packets, txpackets); } }
@@ -1779,14 +1779,18 @@ static bool br_vlan_stats_fill(struct sk_buff *skb,return false;br_vlan_get_stats (v, &stats); -if (nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_RX_BYTES , stats.rx_bytes, +if (nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_RX_BYTES , +u64_stats_read (&stats.rx_bytes),BRIDGE_VLANDB_STATS_PAD ) ||nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_RX_PACKETS , - stats.rx_packets,BRIDGE_VLANDB_STATS_PAD ) || -nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_TX_BYTES , stats.tx_bytes, +u64_stats_read (&stats.rx_packets), +BRIDGE_VLANDB_STATS_PAD ) || +nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_TX_BYTES , +u64_stats_read (&stats.tx_bytes),BRIDGE_VLANDB_STATS_PAD ) ||nla_put_u64_64bit (skb,BRIDGE_VLANDB_STATS_TX_PACKETS , - stats.tx_packets,BRIDGE_VLANDB_STATS_PAD )) +u64_stats_read (&stats.tx_packets), +BRIDGE_VLANDB_STATS_PAD )) goto out_err;nla_nest_end (skb, nest);
net/core/dev.c +9 -9
@@ -10459,23 +10459,23 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,int cpu;for_each_possible_cpu (cpu) { +u64 rx_packets, rx_bytes, tx_packets, tx_bytes;const struct pcpu_sw_netstats *stats; -struct pcpu_sw_netstats tmp;unsigned int start; stats =per_cpu_ptr (netstats, cpu);do { start =u64_stats_fetch_begin_irq (&stats->syncp); - tmp.rx_packets = stats->rx_packets; - tmp.rx_bytes = stats->rx_bytes; - tmp.tx_packets = stats->tx_packets; - tmp.tx_bytes = stats->tx_bytes; + rx_packets =u64_stats_read (&stats->rx_packets); + rx_bytes =u64_stats_read (&stats->rx_bytes); + tx_packets =u64_stats_read (&stats->tx_packets); + tx_bytes =u64_stats_read (&stats->tx_bytes); }while (u64_stats_fetch_retry_irq (&stats->syncp, start)); - s->rx_packets += tmp.rx_packets; - s->rx_bytes += tmp.rx_bytes; - s->tx_packets += tmp.tx_packets; - s->tx_bytes += tmp.tx_bytes; + s->rx_packets += rx_packets; + s->rx_bytes += rx_bytes; + s->tx_packets += tx_packets; + s->tx_bytes += tx_bytes; } }EXPORT_SYMBOL_GPL (dev_fetch_sw_netstats);
net/dsa/slave.c +4 -4
@@ -935,10 +935,10 @@ static void dsa_slave_get_ethtool_stats(struct net_device *dev, s =per_cpu_ptr (dev->tstats, i);do { start =u64_stats_fetch_begin_irq (&s->syncp); - tx_packets = s->tx_packets; - tx_bytes = s->tx_bytes; - rx_packets = s->rx_packets; - rx_bytes = s->rx_bytes; + tx_packets =u64_stats_read (&s->tx_packets); + tx_bytes =u64_stats_read (&s->tx_bytes); + rx_packets =u64_stats_read (&s->rx_packets); + rx_bytes =u64_stats_read (&s->rx_bytes); }while (u64_stats_fetch_retry_irq (&s->syncp, start)); data[0 ] += tx_packets; data[1 ] += tx_bytes;
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 7/9 devlink: adopt u64_stats_t
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
--- net/core/devlink.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-)
net/core/devlink.c +16 -12
@@ -7946,8 +7946,8 @@ static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb, }struct devlink_stats { -u64 rx_bytes; -u64 rx_packets; +u64_stats_t rx_bytes; +u64_stats_t rx_packets;struct u64_stats_sync syncp; };
@@ -8104,12 +8104,12 @@ static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats, cpu_stats =per_cpu_ptr (trap_stats, i);do { start =u64_stats_fetch_begin_irq (&cpu_stats->syncp); - rx_packets = cpu_stats->rx_packets; - rx_bytes = cpu_stats->rx_bytes; + rx_packets =u64_stats_read (&cpu_stats->rx_packets); + rx_bytes =u64_stats_read (&cpu_stats->rx_bytes); }while (u64_stats_fetch_retry_irq (&cpu_stats->syncp, start)); - stats->rx_packets += rx_packets; - stats->rx_bytes += rx_bytes; +u64_stats_add (&stats->rx_packets, rx_packets); +u64_stats_add (&stats->rx_bytes, rx_bytes); } }
@@ -8127,11 +8127,13 @@ devlink_trap_group_stats_put(struct sk_buff *msg,return -EMSGSIZE ;if (nla_put_u64_64bit (msg,DEVLINK_ATTR_STATS_RX_PACKETS , - stats.rx_packets,DEVLINK_ATTR_PAD )) +u64_stats_read (&stats.rx_packets), +DEVLINK_ATTR_PAD )) goto nla_put_failure;if (nla_put_u64_64bit (msg,DEVLINK_ATTR_STATS_RX_BYTES , - stats.rx_bytes,DEVLINK_ATTR_PAD )) +u64_stats_read (&stats.rx_bytes), +DEVLINK_ATTR_PAD )) goto nla_put_failure;nla_nest_end (msg, attr);
@@ -8171,11 +8173,13 @@ static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink, goto nla_put_failure;if (nla_put_u64_64bit (msg,DEVLINK_ATTR_STATS_RX_PACKETS , - stats.rx_packets,DEVLINK_ATTR_PAD )) +u64_stats_read (&stats.rx_packets), +DEVLINK_ATTR_PAD )) goto nla_put_failure;if (nla_put_u64_64bit (msg,DEVLINK_ATTR_STATS_RX_BYTES , - stats.rx_bytes,DEVLINK_ATTR_PAD )) +u64_stats_read (&stats.rx_bytes), +DEVLINK_ATTR_PAD )) goto nla_put_failure;nla_nest_end (msg, attr);
@@ -11641,8 +11645,8 @@ devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats, stats =this_cpu_ptr (trap_stats);u64_stats_update_begin (&stats->syncp); - stats->rx_bytes += skb_len; - stats->rx_packets++; +u64_stats_add (&stats->rx_bytes, skb_len); +u64_stats_inc (&stats->rx_packets);u64_stats_update_end (&stats->syncp); }
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 8/9 drop_monitor: adopt u64_stats_t
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
--- net/core/drop_monitor.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
net/core/drop_monitor.c +9 -9
@@ -55,7 +55,7 @@ static bool monitor_hw;static DEFINE_MUTEX (net_dm_mutex);struct net_dm_stats { -u64 dropped; +u64_stats_t dropped;struct u64_stats_sync syncp; };
@@ -530,7 +530,7 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore, unlock_free:spin_unlock_irqrestore (&data->drop_queue.lock, flags);u64_stats_update_begin (&data->stats.syncp); - data->stats.dropped++; +u64_stats_inc (&data->stats.dropped);u64_stats_update_end (&data->stats.syncp);consume_skb (nskb); }
@@ -985,7 +985,7 @@ net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink, unlock_free:spin_unlock_irqrestore (&hw_data->drop_queue.lock, flags);u64_stats_update_begin (&hw_data->stats.syncp); - hw_data->stats.dropped++; +u64_stats_inc (&hw_data->stats.dropped);u64_stats_update_end (&hw_data->stats.syncp);net_dm_hw_metadata_free (n_hw_metadata); free:
@@ -1432,10 +1432,10 @@ static void net_dm_stats_read(struct net_dm_stats *stats)do { start =u64_stats_fetch_begin_irq (&cpu_stats->syncp); - dropped = cpu_stats->dropped; + dropped =u64_stats_read (&cpu_stats->dropped); }while (u64_stats_fetch_retry_irq (&cpu_stats->syncp, start)); - stats->dropped += dropped; +u64_stats_add (&stats->dropped, dropped); } }
@@ -1451,7 +1451,7 @@ static int net_dm_stats_put(struct sk_buff *msg)return -EMSGSIZE ;if (nla_put_u64_64bit (msg,NET_DM_ATTR_STATS_DROPPED , - stats.dropped,NET_DM_ATTR_PAD )) +u64_stats_read (&stats.dropped),NET_DM_ATTR_PAD )) goto nla_put_failure;nla_nest_end (msg, attr);
@@ -1476,10 +1476,10 @@ static void net_dm_hw_stats_read(struct net_dm_stats *stats)do { start =u64_stats_fetch_begin_irq (&cpu_stats->syncp); - dropped = cpu_stats->dropped; + dropped =u64_stats_read (&cpu_stats->dropped); }while (u64_stats_fetch_retry_irq (&cpu_stats->syncp, start)); - stats->dropped += dropped; +u64_stats_add (&stats->dropped, dropped); } }
@@ -1495,7 +1495,7 @@ static int net_dm_hw_stats_put(struct sk_buff *msg)return -EMSGSIZE ;if (nla_put_u64_64bit (msg,NET_DM_ATTR_STATS_DROPPED , - stats.dropped,NET_DM_ATTR_PAD )) +u64_stats_read (&stats.dropped),NET_DM_ATTR_PAD )) goto nla_put_failure;nla_nest_end (msg, attr);
Signature · 1 line
2.36.1.255.ge46751e96f-goog
PATCH v2 9/9 team: adopt u64_stats_t
In reply to Eric DumazetFrom: Eric Dumazet <[email protected]>
As explained in commit 316580b69d0a ("u64_stats: provide u64_stats_t type") we should use u64_stats_t and related accessors to avoid load/store tearing.
--- drivers/net/team/team.c | 26 +++++++++++++------------- include/linux/if_team.h | 10 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-)
drivers/net/team/team.c +13 -13
@@ -749,10 +749,10 @@ static rx_handler_result_t team_handle_frame(struct sk_buff **pskb) pcpu_stats =this_cpu_ptr (team->pcpu_stats);u64_stats_update_begin (&pcpu_stats->syncp); - pcpu_stats->rx_packets++; - pcpu_stats->rx_bytes += skb->len; +u64_stats_inc (&pcpu_stats->rx_packets); +u64_stats_add (&pcpu_stats->rx_bytes, skb->len);if (skb->pkt_type ==PACKET_MULTICAST ) - pcpu_stats->rx_multicast++; +u64_stats_inc (&pcpu_stats->rx_multicast);u64_stats_update_end (&pcpu_stats->syncp); skb->dev = team->dev;
@@ -1720,8 +1720,8 @@ static netdev_tx_t team_xmit(struct sk_buff *skb, struct net_device *dev) pcpu_stats =this_cpu_ptr (team->pcpu_stats);u64_stats_update_begin (&pcpu_stats->syncp); - pcpu_stats->tx_packets++; - pcpu_stats->tx_bytes += len; +u64_stats_inc (&pcpu_stats->tx_packets); +u64_stats_add (&pcpu_stats->tx_bytes, len);u64_stats_update_end (&pcpu_stats->syncp); }else {this_cpu_inc (team->pcpu_stats->tx_dropped);
@@ -1854,11 +1854,11 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) p =per_cpu_ptr (team->pcpu_stats, i);do { start =u64_stats_fetch_begin_irq (&p->syncp); - rx_packets = p->rx_packets; - rx_bytes = p->rx_bytes; - rx_multicast = p->rx_multicast; - tx_packets = p->tx_packets; - tx_bytes = p->tx_bytes; + rx_packets =u64_stats_read (&p->rx_packets); + rx_bytes =u64_stats_read (&p->rx_bytes); + rx_multicast =u64_stats_read (&p->rx_multicast); + tx_packets =u64_stats_read (&p->tx_packets); + tx_bytes =u64_stats_read (&p->tx_bytes); }while (u64_stats_fetch_retry_irq (&p->syncp, start)); stats->rx_packets += rx_packets;
@@ -1870,9 +1870,9 @@ team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)* rx_dropped, tx_dropped & rx_nohandler are u32, * updated without syncp protection. */ - rx_dropped += p->rx_dropped; - tx_dropped += p->tx_dropped; - rx_nohandler += p->rx_nohandler; + rx_dropped +=READ_ONCE (p->rx_dropped); + tx_dropped +=READ_ONCE (p->tx_dropped); + rx_nohandler +=READ_ONCE (p->rx_nohandler); } stats->rx_dropped = rx_dropped; stats->tx_dropped = tx_dropped;
include/linux/if_team.h +5 -5
@@ -12,11 +12,11 @@#include <uapi/linux/if_team.h> struct team_pcpu_stats { -u64 rx_packets; -u64 rx_bytes; -u64 rx_multicast; -u64 tx_packets; -u64 tx_bytes; +u64_stats_t rx_packets; +u64_stats_t rx_bytes; +u64_stats_t rx_multicast; +u64_stats_t tx_packets; +u64_stats_t tx_bytes;struct u64_stats_sync syncp;u32 rx_dropped;u32 tx_dropped;
Signature · 1 line
2.36.1.255.ge46751e96f-goog
This series was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (master) by Jakub Kicinski <[email protected]>:
Quoted 9 lines · On Wed, 8 Jun 2022 08:46:31 -0700 you wrote:
From: Eric Dumazet <[email protected]>
While KCSAN has not raised any reports yet, we should address the potential load/store tearing problem happening with per cpu stats.
This series is not exhaustive, but hopefully a step in the right direction.
[...]
Here is the summary with links:
- [v2,net-next,1/9] vlan: adopt u64_stats_t
https://git.kernel.org/netdev/net-next/c/09cca53c1656 - [v2,net-next,2/9] ipvlan: adopt u64_stats_t https://git.kernel.org/netdev/net-next/c/5665f48ef309 - [v2,net-next,3/9] sit: use dev_sw_netstats_rx_add() https://git.kernel.org/netdev/net-next/c/3a960ca7f6e5 - [v2,net-next,4/9] ip6_tunnel: use dev_sw_netstats_rx_add() https://git.kernel.org/netdev/net-next/c/afd2051b1840 - [v2,net-next,5/9] wireguard: receive: use dev_sw_netstats_rx_add() https://git.kernel.org/netdev/net-next/c/eeb15885ca30 - [v2,net-next,6/9] net: adopt u64_stats_t in struct pcpu_sw_netstats https://git.kernel.org/netdev/net-next/c/9962acefbcb9 - [v2,net-next,7/9] devlink: adopt u64_stats_t https://git.kernel.org/netdev/net-next/c/958751e0807d - [v2,net-next,8/9] drop_monitor: adopt u64_stats_t https://git.kernel.org/netdev/net-next/c/c6cce71e7468 - [v2,net-next,9/9] team: adopt u64_stats_t https://git.kernel.org/netdev/net-next/c/9ec321aba2ea
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html