u64_stats: Introduce u64_stats_reads()
Subsystems (5)
v2 cover u64_stats: Introduce u64_stats_reads()
On 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() and struct copying should not be considered tear-free. Use u64_stats_reads() instead.
v1: https://lore.kernel.org/r/[email protected]
- rename to u64_stats_reads() to clarify data flow direction
- refine the implementation
- add docs
- fix struct copying too
David Yang (7):
u64_stats: Introduce u64_stats_reads()
u64_stats: Doc incorrect usage with plain variables
net: bridge: mcast: fix memcpy with u64_stats
net: openvswitch: fix load tearing with u64_stats
macsec: fix memcpy with u64_stats
mpls: Fix load tearing with u64_stats
vxlan: vnifilter: fix memcpy with u64_stats
drivers/net/macsec.c | 6 ++-- drivers/net/vxlan/vxlan_vnifilter.c | 2 +- include/linux/u64_stats_sync.h | 56 ++++++++++++++++++++++++----- net/bridge/br_multicast.c | 2 +- net/mpls/af_mpls.c | 2 +- net/openvswitch/datapath.c | 3 +- 6 files changed, 55 insertions(+), 16 deletions(-)
Signature · 1 line
2.51.0
PATCH v2 1/7 u64_stats: Introduce u64_stats_reads()
The following pattern was observed in the code tree:
do {
start = u64_stats_fetch_begin(&pstats->syncp);
memcpy(&temp, &pstats->stats, sizeof(temp));
} while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() or struct copying does not guarantee tear free, although compilers never generate such instructions in practice.
Theoretically the affected code should convert to u64_stats_t, or use atomic operations properly.
However since there are needs to copy chunks of statistics, instead of writing loops everywhere, we provide a safe memcpy() variant for that purpose.
--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
include/linux/u64_stats_sync.h +15 -0
@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); } +static inline void *u64_stats_reads (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len; i +=sizeof (u64_stats_t)) + *(u64 *)(dst + i) =u64_stats_read ((const u64_stats_t *)(src + i)); +return dst; +} +static inline void u64_stats_set (u64_stats_t *p,u64 val) {local64_set (&p->v, val);
@@ -110,6 +118,7 @@ static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, } #else /* 64 bit */ +#include <linux/string.h> typedef struct { u64 v;
@@ -120,6 +129,12 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return p->v; } +static inline void *u64_stats_reads (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +return memcpy (dst, src, len); +} +static inline void u64_stats_set (u64_stats_t *p,u64 val) { p->v = val;
Signature · 1 line
2.51.0
PATCH v2 2/7 u64_stats: Doc incorrect usage with plain variables
In reply to David YangOn 64-bit architectures, u64_stats does rely on the load/store atomicity of 64-bit data.
However, users often mistakenly believe that the helpers could also protect/"lock" plain (64-bit) variables, which can lead to load/store tearing.
Remove the misleading "non atomic operation" comments and add explicit examples of incorrect usage.
Users may also be tempted to use memcpy() or struct copying. Doc the usage of u64_stats_reads() for this case.
--- include/linux/u64_stats_sync.h | 41 ++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-)
include/linux/u64_stats_sync.h +32 -9
@@ -39,21 +39,44 @@* spin_lock_bh(...) or other synchronization to get exclusive access * ... * u64_stats_update_begin(&stats->syncp); -* u64_stats_add(&stats->bytes64, len); // non atomic operation -* u64_stats_inc(&stats->packets64); // non atomic operation +* u64_stats_add(&stats->bytes64, len); +* u64_stats_inc(&stats->packets64); * u64_stats_update_end(&stats->syncp); * * While a consumer (reader) should use following template to get consistent * snapshot for each variable (but no guarantee on several ones) * -* u64 tbytes, tpackets; -* unsigned int start; +* u64 tbytes, tpackets; +* unsigned int start; * -* do { -* start = u64_stats_fetch_begin(&stats->syncp); -* tbytes = u64_stats_read(&stats->bytes64); // non atomic operation -* tpackets = u64_stats_read(&stats->packets64); // non atomic operation -* } while (u64_stats_fetch_retry(&stats->syncp, start)); +* do { +* start = u64_stats_fetch_begin(&stats->syncp); +* tbytes = u64_stats_read(&stats->bytes64); +* tpackets = u64_stats_read(&stats->packets64); +* } while (u64_stats_fetch_retry(&stats->syncp, start)); +* +* Remember point #2: update_begin()/update_end() and +* fetch_begin()/fetch_retry() are no-ops on 64-bit architectures. u64_stats +* _cannot_ be used to protect plain variables against tearing. +* +* u64 stats64, cnt; +* struct { u64_stats_t stats[10]; } st, buf; +* +* u64_stats_update_begin(&stats->syncp); +* stats64 = cnt; // no +* stats64 += cnt; // no +* stats64++; // no +* st = buf; // no +* memcpy(&st, &buf, sizeof(st)); // no +* u64_stats_update_end(&stats->syncp); +* +* do { +* start = u64_stats_fetch_begin(&stats->syncp); +* cnt = stats64; // no +* buf = st; // no +* memcpy(&buf, &st, sizeof(st)); // no +* u64_stats_reads(&buf, &st, sizeof(st)); // use this instead +* } while (u64_stats_fetch_retry(&stats->syncp, start)); * * * Example of use in drivers/net/loopback.c, using per_cpu containers,
Signature · 1 line
2.51.0
PATCH v2 3/7 net: bridge: mcast: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered tear-free. Use u64_stats_reads() instead.
--- net/bridge/br_multicast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/bridge/br_multicast.c +1 -1
@@ -5201,7 +5201,7 @@ void br_multicast_get_stats(const struct net_bridge *br,do { start =u64_stats_fetch_begin (&cpu_stats->syncp); -memcpy (&temp, &cpu_stats->mstats,sizeof (temp)); +u64_stats_reads (&temp, &cpu_stats->mstats,sizeof (temp)); }while (u64_stats_fetch_retry (&cpu_stats->syncp, start));mcast_stats_add_dir (tdst.igmp_v1queries, temp.igmp_v1queries);
Signature · 1 line
2.51.0
PATCH v2 4/7 net: openvswitch: fix load tearing with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Use u64_stats_reads() instead.
--- net/openvswitch/datapath.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
net/openvswitch/datapath.c +2 -1
@@ -770,7 +770,8 @@ static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,do { start =u64_stats_fetch_begin (&percpu_stats->syncp); - local_stats = *percpu_stats; +u64_stats_reads (&local_stats, percpu_stats, +sizeof (local_stats)); }while (u64_stats_fetch_retry (&percpu_stats->syncp, start)); stats->n_hit += local_stats.n_hit;
Signature · 1 line
2.51.0
kernel test robot
build botRe: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats
kernel test robot noticed the following build errors:
Hi David,
kernel test robot noticed the following build errors:
[auto build test ERROR on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/David-Yang/u64_stats-Doc-incorrect-usage-with-plain-variables/20260124-012240 base: net-next/main patch link: https://lore.kernel.org/r/20260123162159.2877941-5-mmyangfl%40gmail.com patch subject: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats config: m68k-randconfig-r073-20260124 (https://download.01.org/0day-ci/archive/20260124/[email protected]/config) compiler: m68k-linux-gcc (GCC) 8.5.0 smatch version: v0.5.0-8994-gd50c5a4c reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260124/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
In file included from <command-line>:
In function 'u64_stats_reads.constprop',
inlined from 'get_dp_stats' at net/openvswitch/datapath.c:773:4:
>> include/linux/compiler_types.h:631:38: error: call to '__compiletime_assert_333' declared with attribute error: BUILD_BUG_ON failed: len % sizeof(u64_stats_t)
_compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
^
include/linux/compiler_types.h:612:4: note: in definition of macro '__compiletime_assert'
prefix ## suffix(); \
^~~~~~
include/linux/compiler_types.h:631:2: note: in expansion of macro '_compiletime_assert'
_compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
^~~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
#define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
^~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:50:2: note: in expansion of macro 'BUILD_BUG_ON_MSG'
BUILD_BUG_ON_MSG(condition, "BUILD_BUG_ON failed: " #condition)
^~~~~~~~~~~~~~~~
include/linux/u64_stats_sync.h:157:2: note: in expansion of macro 'BUILD_BUG_ON'
BUILD_BUG_ON(len % sizeof(u64_stats_t));
^~~~~~~~~~~~vim +/__compiletime_assert_333 +631 include/linux/compiler_types.h
eb5c2d4b45e3d2 Will Deacon 2020-07-21 617 eb5c2d4b45e3d2 Will Deacon 2020-07-21 618 #define _compiletime_assert(condition, msg, prefix, suffix) \ eb5c2d4b45e3d2 Will Deacon 2020-07-21 619 __compiletime_assert(condition, msg, prefix, suffix) eb5c2d4b45e3d2 Will Deacon 2020-07-21 620 eb5c2d4b45e3d2 Will Deacon 2020-07-21 621 /** eb5c2d4b45e3d2 Will Deacon 2020-07-21 622 * compiletime_assert - break build and emit msg if condition is false eb5c2d4b45e3d2 Will Deacon 2020-07-21 623 * @condition: a compile-time constant condition to check eb5c2d4b45e3d2 Will Deacon 2020-07-21 624 * @msg: a message to emit if condition is false eb5c2d4b45e3d2 Will Deacon 2020-07-21 625 * eb5c2d4b45e3d2 Will Deacon 2020-07-21 626 * In tradition of POSIX assert, this macro will break the build if the eb5c2d4b45e3d2 Will Deacon 2020-07-21 627 * supplied condition is *false*, emitting the supplied error message if the eb5c2d4b45e3d2 Will Deacon 2020-07-21 628 * compiler has support to do so. eb5c2d4b45e3d2 Will Deacon 2020-07-21 629 */ eb5c2d4b45e3d2 Will Deacon 2020-07-21 630 #define compiletime_assert(condition, msg) \
eb5c2d4b45e3d2 Will Deacon 2020-07-21 @631 _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)eb5c2d4b45e3d2 Will Deacon 2020-07-21 632
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Ilya Maximets
maintainerRe: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats
In reply to David YangQuoted 14 lines · David Yang
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Useu64_stats_reads()instead.Signed-off-by David Yang <[email protected]>--- net/openvswitch/datapath.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)net/openvswitch/datapath.c +0
-0
net/openvswitch/datapath.c static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats, · line 774
Quoted 6 lines
net/openvswitch/datapath.c +2
-1@@ -770,7 +770,8 @@ static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats, do { start = u64_stats_fetch_begin(&percpu_stats->syncp); - local_stats = *percpu_stats; + u64_stats_reads(&local_stats, percpu_stats, + sizeof(local_stats));
I suppose, you'll either need to use offset of 'syncp' field here, or copy fields one by one, which is not great.
Best regards, Ilya Maximets.
Quoted 3 lines
net/openvswitch/datapath.c +0
-0}while (u64_stats_fetch_retry (&percpu_stats->syncp, start)); stats->n_hit += local_stats.n_hit;
Re: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats
In reply to David YangQuoted 3 lines · On Sat, 24 Jan 2026 00:21:36 +0800 David Yang <[email protected]> wrote:
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Useu64_stats_reads()instead.
Except that the compiler doesn't ever generate 'tearing accesses' for aligned 64bit accesses on any 64bit architecture. Similarly memcpy() won't generate problematic accesses.
The problem is purely theoretical - the C language lets the compiler split accesses, but it doesn't.
David
Quoted 20 lines
Signed-off-by David Yang <[email protected]>--- net/openvswitch/datapath.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)net/openvswitch/datapath.c +2
-1@@ -770,7 +770,8 @@ static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,do { start =u64_stats_fetch_begin (&percpu_stats->syncp); - local_stats = *percpu_stats; +u64_stats_reads (&local_stats, percpu_stats, +sizeof (local_stats)); }while (u64_stats_fetch_retry (&percpu_stats->syncp, start)); stats->n_hit += local_stats.n_hit;
Re: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats
Quoted 8 lines · David Laight
Quoted 3 lines · On Sat, 24 Jan 2026 00:21:36 +0800 David Yang <[email protected]> wrote:
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Useu64_stats_reads()instead.
Quoted 6 lines
Except that the compiler doesn't ever generate 'tearing accesses' for aligned 64bit accesses on any 64bit architecture. Similarly
memcpy()won't generate problematic accesses.The problem is purely theoretical - the C language lets the compiler split accesses, but it doesn't.
Yeah, although we still have races that KCSAN can detect.
data_race() or READ_ONCE() would be necessary to avoid noisy KCSAN reports.
While many LCSAN reports are boring, some of them point to real bugs.
Re: [PATCH net-next v2 4/7] net: openvswitch: fix load tearing with u64_stats
Quoted 14 lines · On Mon, 26 Jan 2026 19:35:38 +0100 Eric Dumazet <[email protected]> wrote:
Quoted 12 lines · David Laight
Quoted 3 lines · On Sat, 24 Jan 2026 00:21:36 +0800 David Yang <[email protected]> wrote:
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Useu64_stats_reads()instead.Except that the compiler doesn't ever generate 'tearing accesses' for aligned 64bit accesses on any 64bit architecture. Similarly
memcpy()won't generate problematic accesses.
Quoted 9 lines
Quoted 2 lines
The problem is purely theoretical - the C language lets the compiler split accesses, but it doesn't.
Yeah, although we still have races that KCSAN can detect.
data_race()orREAD_ONCE()would be necessary to avoid noisy KCSAN reports.While many KCSAN reports are boring, some of them point to real bugs.
Could something be put in u64_stats_fetch_begin() to stop KCSAN bleating?
With the way READ_ONCE and WRITE_ONCE now get enforced I do wonder if some data shouldn't just be marked 'volatile'?
That would give 'non-tearing' accesses, but without the inter-cpu ordering that (I think) READ/WRITE_ONCE also give.
For stats counters that is enough.
David
PATCH v2 5/7 macsec: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered tear-free. Use u64_stats_reads() instead.
--- drivers/net/macsec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/macsec.c +3 -3
@@ -2806,7 +2806,7 @@ static void get_rx_sc_stats(struct net_device *dev, stats =per_cpu_ptr (rx_sc->stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_reads (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->InOctetsValidated += tmp.InOctetsValidated;
@@ -2887,7 +2887,7 @@ static void get_tx_sc_stats(struct net_device *dev, stats =per_cpu_ptr (macsec_priv (dev)->secy.tx_sc.stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_reads (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->OutPktsProtected += tmp.OutPktsProtected;
@@ -2943,7 +2943,7 @@ static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum) stats =per_cpu_ptr (macsec_priv (dev)->stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_reads (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->OutPktsUntagged += tmp.OutPktsUntagged;
Signature · 1 line
2.51.0
PATCH v2 6/7 mpls: Fix load tearing with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. struct copying should not be considered tear-free. Use u64_stats_reads() instead.
--- net/mpls/af_mpls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/mpls/af_mpls.c +1 -1
@@ -1097,7 +1097,7 @@ static void mpls_get_stats(struct mpls_dev *mdev, p =per_cpu_ptr (mdev->stats, i);do { start =u64_stats_fetch_begin (&p->syncp); - local = p->stats; +u64_stats_reads (&local, &p->stats,sizeof (local)); }while (u64_stats_fetch_retry (&p->syncp, start)); stats->rx_packets += local.rx_packets;
Signature · 1 line
2.51.0
PATCH v2 7/7 vxlan: vnifilter: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered tear-free. Use u64_stats_reads() instead.
--- drivers/net/vxlan/vxlan_vnifilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/net/vxlan/vxlan_vnifilter.c +1 -1
@@ -126,7 +126,7 @@ static void vxlan_vnifilter_stats_get(const struct vxlan_vni_node *vninode, pstats =per_cpu_ptr (vninode->stats, i);do { start =u64_stats_fetch_begin (&pstats->syncp); -memcpy (&temp, &pstats->stats,sizeof (temp)); +u64_stats_reads (&temp, &pstats->stats,sizeof (temp)); }while (u64_stats_fetch_retry (&pstats->syncp, start)); dest->rx_packets += temp.rx_packets;
Signature · 1 line
2.51.0