u64_stats: Introduce u64_stats_copy()
Subsystems (4)
cover u64_stats: Introduce u64_stats_copy()
On 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered atomic against u64 values. Use u64_stats_copy() instead.
David Yang (4):
u64_stats: Introduce u64_stats_copy()
net: bridge: mcast: fix memcpy with u64_stats
macsec: fix memcpy with u64_stats
vxlan: vnifilter: fix memcpy with u64_stats
drivers/net/macsec.c | 6 +++--- drivers/net/vxlan/vxlan_vnifilter.c | 2 +- include/linux/u64_stats_sync.h | 15 +++++++++++++++ net/bridge/br_multicast.c | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-)
Signature · 1 line
2.51.0
PATCH 1/4 u64_stats: Introduce u64_stats_copy()
The following (anti-)pattern was observed in the code tree:
do {
start = u64_stats_fetch_begin(&pstats->syncp);
memcpy(&temp, &pstats->stats, sizeof(temp));
} while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing, especially for memcpy(), for which arches may provide their highly-optimized implements.
In theory the affected code should convert to u64_stats_t, or use READ_ONCE()/WRITE_ONCE() properly.
However since there are needs to copy chunks of statistics, instead of writing loops at random places, we provide a safe memcpy() variant for u64_stats.
--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
include/linux/u64_stats_sync.h +15 -0
@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); } +static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len /sizeof (u64_stats_t); i++) + ((u64 *)dst)[i] =local64_read (&((local64_t *)src)[i]); +return dst; +} +static inline void u64_stats_set (u64_stats_t *p,u64 val) {local64_set (&p->v, val);
@@ -110,6 +118,7 @@ static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, } #else /* 64 bit */ +#include <linux/string.h> typedef struct { u64 v;
@@ -120,6 +129,12 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return p->v; } +static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +return memcpy (dst, src, len); +} +static inline void u64_stats_set (u64_stats_t *p,u64 val) { p->v = val;
Signature · 1 line
2.51.0
Sabrina Dubroca
maintainerQuoted 28 lines · David Yang
The following (anti-)pattern was observed in the code tree:
do { start = u64_stats_fetch_begin(&pstats->syncp); memcpy(&temp, &pstats->stats, sizeof(temp)); } while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing, especially formemcpy(), for which arches may provide their highly-optimized implements.In theory the affected code should convert to
u64_stats_t, or use READ_ONCE()/WRITE_ONCE() properly.However since there are needs to copy chunks of statistics, instead of writing loops at random places, we provide a safe
memcpy()variant foru64_stats.Signed-off-by David Yang <[email protected]>--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)include/linux/u64_stats_sync.h +0
-0
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 86
Quoted 8 lines
include/linux/u64_stats_sync.h +5
-0@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); } +static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len /sizeof (u64_stats_t); i++) + ((u64 *)dst)[i] =local64_read (&((local64_t *)src)[i]);
Maybe u64_stats_read/u64_stats_t instead of local64_read/local64_t?
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 88
Quoted 2 lines
include/linux/u64_stats_sync.h +2
-0+return dst; +}
Since this new helper is always used within a u64_stats_fetch_begin/u64_stats_fetch_retry loop, maybe it would be nicer to push the retry loop into the helper as well? Not a strong opinion. It would be a bit "simpler" for the callers, but your current proposal has the advantage of looking like memcpy(), and of also looking (for the caller) like other retry loops fetching each counter explicitly.
Either way, I think extending the "Usage" section of the big comment at the top of the file with this new helper would be nice.
Signature · 1 line
Sabrina
Quoted 33 lines · Sabrina Dubroca
Quoted 31 lines · David Yang
The following (anti-)pattern was observed in the code tree:
do { start = u64_stats_fetch_begin(&pstats->syncp); memcpy(&temp, &pstats->stats, sizeof(temp)); } while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing, especially formemcpy(), for which arches may provide their highly-optimized implements.In theory the affected code should convert to
u64_stats_t, or use READ_ONCE()/WRITE_ONCE() properly.However since there are needs to copy chunks of statistics, instead of writing loops at random places, we provide a safe
memcpy()variant foru64_stats.Signed-off-by David Yang <[email protected]>--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)include/linux/u64_stats_sync.h +0
-0@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); }
Quoted 8 lines
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 86
Quoted 5 lines
include/linux/u64_stats_sync.h +5
-0+static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len /sizeof (u64_stats_t); i++) + ((u64 *)dst)[i] =local64_read (&((local64_t *)src)[i]);Maybe u64_stats_read/u64_stats_t instead of local64_read/local64_t?
I think casting to u64_stats_t is a bit overkill here since we accept const void * and we are the actual implementation.
Again, they should convert to u64_stats_t, and this solution already implies that u64_stats_t is binary compatible with u64. I've already sent several patches related to that, but that's another issue.
Quoted 11 lines
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 88
Quoted 2 lines
include/linux/u64_stats_sync.h +2
-0+return dst; +}Since this new helper is always used within a u64_stats_fetch_begin/u64_stats_fetch_retry loop, maybe it would be nicer to push the retry loop into the helper as well? Not a strong opinion. It would be a bit "simpler" for the callers, but your current proposal has the advantage of looking like
memcpy(), and of also looking (for the caller) like other retry loops fetching each counter explicitly.
The callers may want to copy other discontinuous data as well, albeit no one did it then.
do {
start = u64_stats_fetch_begin(&pstats->syncp);
memcpy(...);
u64_stats_read(...);
} while (u64_stats_fetch_retry(&pstats->syncp, start));It would be redundant to provide two variants of the function. Moreover, callers can (and already) invent their own reader/writer helpers, for example
#define SLIC_GET_STATS_COUNTER(newst, st, counter) \
{ \
unsigned int start; \
do { \
start = u64_stats_fetch_begin(&(st)->syncp); \
newst = u64_stats_read(&(st)->counter); \
} while (u64_stats_fetch_retry(&(st)->syncp, start)); \
}Quoted 3 lines
Either way, I think extending the "Usage" section of the big comment at the top of the file with this new helper would be nice.
I think callers should avoid memcpy() eventually, and they almost certainly copied more data than what they need. However, I took a look at some instances, and it would be non trivial to modify those drivers.
Quoted 2 lines
--
Sabrina
Sabrina Dubroca
maintainerQuoted 43 lines · Yangfl
Quoted 33 lines · Sabrina Dubroca
Quoted 31 lines · David Yang
The following (anti-)pattern was observed in the code tree:
do { start = u64_stats_fetch_begin(&pstats->syncp); memcpy(&temp, &pstats->stats, sizeof(temp)); } while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing, especially formemcpy(), for which arches may provide their highly-optimized implements.In theory the affected code should convert to
u64_stats_t, or use READ_ONCE()/WRITE_ONCE() properly.However since there are needs to copy chunks of statistics, instead of writing loops at random places, we provide a safe
memcpy()variant foru64_stats.Signed-off-by David Yang <[email protected]>--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)include/linux/u64_stats_sync.h +0
-0@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); }Quoted 8 lines
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 86
Quoted 5 lines
include/linux/u64_stats_sync.h +5
-0+static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len /sizeof (u64_stats_t); i++) + ((u64 *)dst)[i] =local64_read (&((local64_t *)src)[i]);Maybe u64_stats_read/u64_stats_t instead of local64_read/local64_t?
Quoted 2 lines
I think casting to
u64_stats_tis a bit overkill here since we accept const void * and we are the actual implementation.
It would be a bit more consistent. Just within this function you have 2 lines using u64_stats_t and the 3rd uses local64_t. And reusing types/helpers within a similar context doesn't seem overkill.
[...]
Quoted 11 lines
Quoted 8 lines
Since this new helper is always used within a u64_stats_fetch_begin/u64_stats_fetch_retry loop, maybe it would be nicer to push the retry loop into the helper as well? Not a strong opinion. It would be a bit "simpler" for the callers, but your current proposal has the advantage of looking like
memcpy(), and of also looking (for the caller) like other retry loops fetching each counter explicitly.The callers may want to copy other discontinuous data as well, albeit no one did it then.
I'm not sure why they would. I think the main point of using memcpy is "I don't want to copy each counter by name one by one", and possibly "I don't want to have to patch this code as well if we add a new counter". If you already have a batch copy for a bunch of counters, it's usually easier to add others in a contiguous block.
Quoted 4 lines
It would be redundant to provide two variants of the function. Moreover, callers can (and already) invent their own reader/writer helpers, for example
Quoted 8 lines
#define SLIC_GET_STATS_COUNTER(newst, st, counter) \
{ \
unsigned int start; \
do { \
start = u64_stats_fetch_begin(&(st)->syncp); \
newst = u64_stats_read(&(st)->counter); \
} while (u64_stats_fetch_retry(&(st)->syncp, start)); \
}Probably because the retry loop is a bit cumbersome and they'd rather not c/p it everywhere, and see it in the middle of whatever function needs it.
Quoted 8 lines
Quoted 3 lines
Either way, I think extending the "Usage" section of the big comment at the top of the file with this new helper would be nice.
I think callers should avoid
memcpy()eventually, and they almost certainly copied more data than what they need. However, I took a look at some instances, and it would be non trivial to modify those drivers.
I'm not asking you to fix something else. But for example commit 316580b69d0a ("u64_stats: provide u64_stats_t type") modified the bit of documentation we have at the top of the file to help developers who want to use this API. This patch is introducing a new function and should also describe how to use it, so that new users aren't tempted to re-introduce a memcpy.
Signature · 1 line
Sabrina
Quoted 63 lines · Sabrina Dubroca
Quoted 43 lines · Yangfl
Quoted 33 lines · Sabrina Dubroca
Quoted 31 lines · David Yang
The following (anti-)pattern was observed in the code tree:
do { start = u64_stats_fetch_begin(&pstats->syncp); memcpy(&temp, &pstats->stats, sizeof(temp)); } while (u64_stats_fetch_retry(&pstats->syncp, start));On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing, especially formemcpy(), for which arches may provide their highly-optimized implements.In theory the affected code should convert to
u64_stats_t, or use READ_ONCE()/WRITE_ONCE() properly.However since there are needs to copy chunks of statistics, instead of writing loops at random places, we provide a safe
memcpy()variant foru64_stats.Signed-off-by David Yang <[email protected]>--- include/linux/u64_stats_sync.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+)include/linux/u64_stats_sync.h +0
-0@@ -79,6 +79,14 @@ static inline u64 u64_stats_read(const u64_stats_t *p)return local64_read (&p->v); }Quoted 8 lines
include/linux/u64_stats_sync.h static inline u64 u64_stats_read(const u64_stats_t *p) · line 86
Quoted 5 lines
include/linux/u64_stats_sync.h +5
-0+static inline void *u64_stats_copy (void *dst,const void *src,size_t len) +{ +BUILD_BUG_ON (len %sizeof (u64_stats_t)); +for (size_t i =0 ; i < len /sizeof (u64_stats_t); i++) + ((u64 *)dst)[i] =local64_read (&((local64_t *)src)[i]);Maybe u64_stats_read/u64_stats_t instead of local64_read/local64_t?
Quoted 2 lines
I think casting to
u64_stats_tis a bit overkill here since we accept const void * and we are the actual implementation.It would be a bit more consistent. Just within this function you have 2 lines using
u64_stats_tand the 3rd useslocal64_t. And reusing types/helpers within a similar context doesn't seem overkill.[...]
Quoted 9 lines
Quoted 8 lines
Since this new helper is always used within a u64_stats_fetch_begin/u64_stats_fetch_retry loop, maybe it would be nicer to push the retry loop into the helper as well? Not a strong opinion. It would be a bit "simpler" for the callers, but your current proposal has the advantage of looking like
memcpy(), and of also looking (for the caller) like other retry loops fetching each counter explicitly.
Quoted 9 lines
Quoted 2 lines
The callers may want to copy other discontinuous data as well, albeit no one did it then.
I'm not sure why they would. I think the main point of using memcpy is "I don't want to copy each counter by name one by one", and possibly "I don't want to have to patch this code as well if we add a new counter". If you already have a batch copy for a bunch of counters, it's usually easier to add others in a contiguous block.
While I agree with your statement, I don't think it's a good idea to push the retry loop into the helper. u64_stats_copy(syncp, dst, src, len) would be a strange API, while no others accept syncp argument. Also, it would give a false appearance to those who read the driver code, since it does not involve fetch_begin()/fetch_retry() explicitly.
In my opinion, it would be better to introduce a for-like macro #define with_u64_stats_fetch(syncp), eliminating two function calls as well as one variable declaration altogether.
PATCH 2/4 net: bridge: mcast: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered atomic against u64 values. Use u64_stats_copy() instead.
--- net/bridge/br_multicast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/bridge/br_multicast.c +1 -1
@@ -5201,7 +5201,7 @@ void br_multicast_get_stats(const struct net_bridge *br,do { start =u64_stats_fetch_begin (&cpu_stats->syncp); -memcpy (&temp, &cpu_stats->mstats,sizeof (temp)); +u64_stats_copy (&temp, &cpu_stats->mstats,sizeof (temp)); }while (u64_stats_fetch_retry (&cpu_stats->syncp, start));mcast_stats_add_dir (tdst.igmp_v1queries, temp.igmp_v1queries);
Signature · 1 line
2.51.0
PATCH 3/4 macsec: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered atomic against u64 values. Use u64_stats_copy() instead.
--- drivers/net/macsec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
drivers/net/macsec.c +3 -3
@@ -2806,7 +2806,7 @@ static void get_rx_sc_stats(struct net_device *dev, stats =per_cpu_ptr (rx_sc->stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_copy (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->InOctetsValidated += tmp.InOctetsValidated;
@@ -2887,7 +2887,7 @@ static void get_tx_sc_stats(struct net_device *dev, stats =per_cpu_ptr (macsec_priv (dev)->secy.tx_sc.stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_copy (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->OutPktsProtected += tmp.OutPktsProtected;
@@ -2943,7 +2943,7 @@ static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum) stats =per_cpu_ptr (macsec_priv (dev)->stats, cpu);do { start =u64_stats_fetch_begin (&stats->syncp); -memcpy (&tmp, &stats->stats,sizeof (tmp)); +u64_stats_copy (&tmp, &stats->stats,sizeof (tmp)); }while (u64_stats_fetch_retry (&stats->syncp, start)); sum->OutPktsUntagged += tmp.OutPktsUntagged;
Signature · 1 line
2.51.0
PATCH 4/4 vxlan: vnifilter: fix memcpy with u64_stats
In reply to David YangOn 64bit arches, struct u64_stats_sync is empty and provides no help against load/store tearing. memcpy() should not be considered atomic against u64 values. Use u64_stats_copy() instead.
--- drivers/net/vxlan/vxlan_vnifilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/net/vxlan/vxlan_vnifilter.c +1 -1
@@ -126,7 +126,7 @@ static void vxlan_vnifilter_stats_get(const struct vxlan_vni_node *vninode, pstats =per_cpu_ptr (vninode->stats, i);do { start =u64_stats_fetch_begin (&pstats->syncp); -memcpy (&temp, &pstats->stats,sizeof (temp)); +u64_stats_copy (&temp, &pstats->stats,sizeof (temp)); }while (u64_stats_fetch_retry (&pstats->syncp, start)); dest->rx_packets += temp.rx_packets;
Signature · 1 line
2.51.0
Quoted 3 lines · David Yang
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing.memcpy()should not be considered atomic against u64 values. Useu64_stats_copy()instead.
The existing memcpy() does seem problematic (even if in practice it's not) and the proposed solution in patch #1 seems OK to me given that all the callers only pass structures containing 64 bit counters. Couldn't find any more instances of this pattern.
Sabrina Dubroca
maintainerQuoted 9 lines · Ido Schimmel
Quoted 3 lines · David Yang
On 64bit arches, struct
u64_stats_syncis empty and provides no help against load/store tearing.memcpy()should not be considered atomic against u64 values. Useu64_stats_copy()instead.The existing
memcpy()does seem problematic (even if in practice it's not) and the proposed solution in patch #1 seems OK to me given that all the callers only pass structures containing 64 bit counters. Couldn't find any more instances of this pattern.
No direct instances using memcpy, but do we need to also full structs copied within a u64_stats_fetch_begin/u64_stats_fetch_retry loop?
// net/mpls/af_mpls.c
static void mpls_get_stats(struct mpls_dev *mdev,
struct mpls_link_stats *stats)
{
[...]
for_each_possible_cpu(i) {
struct mpls_link_stats local;
unsigned int start;
p = per_cpu_ptr(mdev->stats, i);
do {
start = u64_stats_fetch_begin(&p->syncp);
local = p->stats;
} while (u64_stats_fetch_retry(&p->syncp, start));[...]
// net/openvswitch/datapath.c
static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
struct ovs_dp_megaflow_stats *mega_stats)
{
[...]
for_each_possible_cpu(i) {
const struct dp_stats_percpu *percpu_stats;
struct dp_stats_percpu local_stats;
unsigned int start;
percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
do {
start = u64_stats_fetch_begin(&percpu_stats->syncp);
local_stats = *percpu_stats;
} while (u64_stats_fetch_retry(&percpu_stats->syncp, start));[...]
And if not: can't we just use the same pattern for those other cases that this series is touching?
Signature · 1 line
Sabrina
Ido Schimmel
maintainerQuoted 2 lines · Sabrina Dubroca
No direct instances using memcpy, but do we need to also full structs copied within a u64_stats_fetch_begin/u64_stats_fetch_retry loop?
My understanding is that we cannot rely on the compiler to perform the copy in any particular way. With the suggested helper it is at least clear how the copy is done.
Sabrina Dubroca
maintainerQuoted 7 lines · Ido Schimmel
Quoted 2 lines · Sabrina Dubroca
No direct instances using memcpy, but do we need to also full structs copied within a u64_stats_fetch_begin/u64_stats_fetch_retry loop?
My understanding is that we cannot rely on the compiler to perform the copy in any particular way. With the suggested helper it is at least clear how the copy is done.
Ok, thanks. So those (and a few more similar things in drivers/net) should also be switched to this new helper.
Signature · 1 line
Sabrina