urcu: avoid false sharing for rcu_gp_ctr
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 9 lines · * Lai Jiangshan ([email protected]) wrote:
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but
they always be accessed at almost the same time, so we also movercu_gp_futexto the cacheline ofrcu_gp_ctrto reduce the cacheline-usage or cache-missing of read site.
Hi Lai,
I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 195 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219nr_ops2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478nr_ops1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334nr_ops1952536372after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248nr_ops3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809nr_ops3401767295Singed-by Lai Jiangshan <[email protected]>---
urcu.c +12
-21@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader and the @@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +348,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before -* committing next rcu_gp_ctr update to memory. Failure to do so could +* committing next rcu_gp.ctr update to memory. Failure to do so could * result in the writer waiting forever while new readers are always * accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();@@ -372,13 +363,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent +* Must commit rcu_gp.ctr update to memory before waiting for quiescent * state. Failure to do so could result in the writer waiting forever * while new readers are always accessing data (no progress). Enforce -* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr. +* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr. */ cmm_barrier ();urcu/static/urcu.h +20
-14@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -231,16 +239,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr, tmp +RCU_GP_COUNT );
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20121207163234.GB11975@Krystal
we can define rcu_gp_ctr and registry with aligned attribute, but it is not reliable way
We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))
Quoted 270 lines · Mathieu Desnoyers
Quoted 7 lines · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orcache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.Thoughts ?
Thanks,
Mathieu
Quoted 8 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809 nr_ops 3401767295Quoted 9 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 10 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader and the
urcu.c static void wait_for_readers(struct cds_list_head
Quoted 12 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c void synchronize_rcu(void) · line 350
Quoted 12 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +348,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before
urcu.c void synchronize_rcu(void) · line 360
Quoted 1 line
urcu.c +0
-1-* committing next rcu_gp_ctr update to memory. Failure to do so could
urcu.c void synchronize_rcu(void) · line 351
Quoted 1 line
urcu.c +1
-0+* committing next rcu_gp.ctr update to memory. Failure to do so could
urcu.c void synchronize_rcu(void) · line 352
Quoted 1 line
urcu.c +0
-0* result in the writer waiting forever while new readers are always
Quoted 7 lines
urcu.c +1
-1* accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();urcu.c void synchronize_rcu(void) · line 378
Quoted 8 lines
urcu.c +1
-2@@ -372,13 +363,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent
urcu.c void synchronize_rcu(void) · line 369
Quoted 1 line
urcu.c +1
-0+* Must commit rcu_gp.ctr update to memory before waiting for quiescent
urcu.c void synchronize_rcu(void) · line 370
Quoted 1 line
urcu.c +0
-0* state. Failure to do so could result in the writer waiting forever
urcu.c void synchronize_rcu(void) · line 371
Quoted 1 line
urcu.c +0
-0* while new readers are always accessing data (no progress). Enforce
urcu.c void synchronize_rcu(void) · line 381
Quoted 1 line
urcu.c +0
-1-* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
urcu.c void synchronize_rcu(void) · line 372
Quoted 1 line
urcu.c +1
-0+* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
Quoted 8 lines
urcu.c +0
-0*/ cmm_barrier ();urcu/static/urcu.h +0
-0urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 218
Quoted 6 lines
urcu/static/urcu.h +0
-3@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies
urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 220
Quoted 8 lines
urcu/static/urcu.h +5
-3-* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
Quoted 13 lines
urcu/static/urcu.h +9
-0+* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ urcu/static/urcu.h struct rcu_reader { · line 243
Quoted 7 lines
urcu/static/urcu.h +0
-2@@ -231,16 +239,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
urcu/static/urcu.h static inline enum rcu_state
Quoted 13 lines
urcu/static/urcu.h +3
-3*/ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +262,13 @@ static inline enum rcu_state
rcu_reader_state(unsigned long *ctr)urcu/static/urcu.h static inline enum rcu_state · line 272
Quoted 13 lines
urcu/static/urcu.h +2
-2v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of
urcu/static/urcu.h static inline enum rcu_state · line 273
Quoted 1 line
urcu/static/urcu.h +0
-0* _rcu_read_lock() nesting, and a lower-order bit that contains either zero
urcu/static/urcu.h static inline enum rcu_state
Quoted 2 lines
urcu/static/urcu.h +0
-0* or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +277,7 @@ static inline enum rcu_state
rcu_reader_state(unsigned long *ctr)urcu/static/urcu.h static inline enum rcu_state · line 274
Quoted 4 lines
urcu/static/urcu.h +0
-1static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { - _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,
_CMM_LOAD_SHARED(rcu_gp_ctr));urcu/static/urcu.h static inline enum rcu_state · line 280
Quoted 1 line
urcu/static/urcu.h +1
-0+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,
_CMM_LOAD_SHARED(rcu_gp.ctr));urcu/static/urcu.h static inline enum rcu_state · line 283
Quoted 3 lines
urcu/static/urcu.h +0
-0smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp +
RCU_GP_COUNT);--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
Quoted 5 lines
_______________________________________________
lttng-dev mailing list
[email protected] <javascript:;>
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
A message that is not in the archive (sent off-list or not yet received): CACvQF50Y2wC3t=VMJ==574aHVvod+ipGA1K5hR97r_dRbcegkw@mail.gmail.com
Quoted 6 lines · * Lai Jiangshan ([email protected]) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it is not reliable wayWe need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))
In which situation would this be unreliable ?
Thanks,
Mathieu
Quoted 278 lines · Mathieu Desnoyers
Quoted 276 lines · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orcache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.Thoughts ?
Thanks,
Mathieu
Quoted 8 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809 nr_ops 3401767295Quoted 9 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 10 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader and the
urcu.c static void wait_for_readers(struct cds_list_head
Quoted 12 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c void synchronize_rcu(void) · line 350
Quoted 12 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +348,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before
urcu.c void synchronize_rcu(void) · line 360
Quoted 1 line
urcu.c +0
-1-* committing next rcu_gp_ctr update to memory. Failure to do so could
urcu.c void synchronize_rcu(void) · line 351
Quoted 1 line
urcu.c +1
-0+* committing next rcu_gp.ctr update to memory. Failure to do so could
urcu.c void synchronize_rcu(void) · line 352
Quoted 1 line
urcu.c +0
-0* result in the writer waiting forever while new readers are always
Quoted 7 lines
urcu.c +1
-1* accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();urcu.c void synchronize_rcu(void) · line 378
Quoted 8 lines
urcu.c +1
-2@@ -372,13 +363,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent
urcu.c void synchronize_rcu(void) · line 369
Quoted 1 line
urcu.c +1
-0+* Must commit rcu_gp.ctr update to memory before waiting for quiescent
urcu.c void synchronize_rcu(void) · line 370
Quoted 1 line
urcu.c +0
-0* state. Failure to do so could result in the writer waiting forever
urcu.c void synchronize_rcu(void) · line 371
Quoted 1 line
urcu.c +0
-0* while new readers are always accessing data (no progress). Enforce
urcu.c void synchronize_rcu(void) · line 381
Quoted 1 line
urcu.c +0
-1-* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
urcu.c void synchronize_rcu(void) · line 372
Quoted 1 line
urcu.c +1
-0+* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
Quoted 8 lines
urcu.c +0
-0*/ cmm_barrier ();urcu/static/urcu.h +0
-0urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 218
Quoted 6 lines
urcu/static/urcu.h +0
-3@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies
urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 220
Quoted 8 lines
urcu/static/urcu.h +5
-3-* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
Quoted 13 lines
urcu/static/urcu.h +9
-0+* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ urcu/static/urcu.h struct rcu_reader { · line 243
Quoted 7 lines
urcu/static/urcu.h +0
-2@@ -231,16 +239,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
urcu/static/urcu.h static inline enum rcu_state
Quoted 13 lines
urcu/static/urcu.h +3
-3*/ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +262,13 @@ static inline enum rcu_state
rcu_reader_state(unsigned long *ctr)urcu/static/urcu.h static inline enum rcu_state · line 272
Quoted 13 lines
urcu/static/urcu.h +2
-2v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of
urcu/static/urcu.h static inline enum rcu_state · line 273
Quoted 1 line
urcu/static/urcu.h +0
-0* _rcu_read_lock() nesting, and a lower-order bit that contains either zero
urcu/static/urcu.h static inline enum rcu_state
Quoted 2 lines
urcu/static/urcu.h +0
-0* or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +277,7 @@ static inline enum rcu_state
rcu_reader_state(unsigned long *ctr)urcu/static/urcu.h static inline enum rcu_state · line 274
Quoted 4 lines
urcu/static/urcu.h +0
-1static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { - _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,
_CMM_LOAD_SHARED(rcu_gp_ctr));urcu/static/urcu.h static inline enum rcu_state · line 280
Quoted 1 line
urcu/static/urcu.h +1
-0+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,
_CMM_LOAD_SHARED(rcu_gp.ctr));urcu/static/urcu.h static inline enum rcu_state · line 283
Quoted 3 lines
urcu/static/urcu.h +0
-0smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp +
RCU_GP_COUNT);--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com_______________________________________________
lttng-dev mailing list
[email protected] <javascript:;>
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20121207165748.GA12338@Krystal
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
int a;
int b __attribute__((aligned));
int c;
b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
Quoted 151 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 30 lines
Quoted 30 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>
Quoted 9 lines
Quoted 2 lines
--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
A message that is not in the archive (sent off-list or not yet received): CACvQF50V_9T6T3j4AwbdBuEiHHbcxdNGgP1WF5T4OHpXZyMgEA@mail.gmail.com
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
Quoted 6 lines
int a;
int b __attribute__((aligned));
int c;b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:
typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));
I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.
I guess this is a case where we could bump the library version number and do things properly.
Let's think a bit more about it, anyone else has comments on this ?
Thanks,
Mathieu
Quoted 162 lines
Quoted 162 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 27 lines
Quoted 27 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 5 lines
Quoted 5 lines
urcu.c static void wait_for_readers(struct cds_list_head · line 282
Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20121207172252.GA12525@Krystal
Quoted 30 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
Quoted 6 lines
int a;
int b __attribute__((aligned));
int c;b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
So if our goal is to have
rcu_gp_ctrandrcu_gp_futexon the same cache line, which is different from that of the registry, we could do:typeA
rcu_gp_ctr__attribute__((aligned(...)));
typeBrcu_gp_futex;
typeC registry __attribute__((aligned(...)));
Quoted 8 lines
I would expect the compiler won't typically reorder
rcu_gp_futexand registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.I guess this is a case where we could bump the library version number and do things properly.
Let's think a bit more about it, anyone else has comments on this ?
If you really want the alignment and padding, they should go into a structure. ;-)
Thanx, Paul
Quoted 173 lines
Thanks,
Mathieu
Quoted 162 lines
Quoted 162 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 27 lines
Quoted 27 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 5 lines
Quoted 5 lines
urcu.c static void wait_for_readers(struct cds_list_head · line 282
Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 35 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 34 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
Quoted 6 lines
int a;
int b __attribute__((aligned));
int c;b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
So if our goal is to have
rcu_gp_ctrandrcu_gp_futexon the same cache line, which is different from that of the registry, we could do:typeA
rcu_gp_ctr__attribute__((aligned(...)));
typeBrcu_gp_futex;
typeC registry __attribute__((aligned(...)));I would expect the compiler won't typically reorder
rcu_gp_futexand registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.
Quoted 7 lines
Quoted 4 lines
I guess this is a case where we could bump the library version number and do things properly.
Let's think a bit more about it, anyone else has comments on this ?
If you really want the alignment and padding, they should go into a structure. ;-)
OK, so we would bump to liburcu2 then for the 0.8 release ?
Thanks,
Mathieu
Quoted 177 lines
Thanx, Paul
Quoted 173 lines
Thanks,
Mathieu
Quoted 162 lines
Quoted 162 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 27 lines
Quoted 27 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 5 lines
Quoted 5 lines
urcu.c static void wait_for_readers(struct cds_list_head · line 282
Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20121210180111.GA11209@Krystal
Quoted 39 lines · Mathieu Desnoyers
Quoted 38 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 37 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
Quoted 6 lines
int a;
int b __attribute__((aligned));
int c;b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
So if our goal is to have
rcu_gp_ctrandrcu_gp_futexon the same cache line, which is different from that of the registry, we could do:typeA
rcu_gp_ctr__attribute__((aligned(...)));
typeBrcu_gp_futex;
typeC registry __attribute__((aligned(...)));I would expect the compiler won't typically reorder
rcu_gp_futexand registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.I guess this is a case where we could bump the library version number and do things properly.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Let's think a bit more about it, anyone else has comments on this ?
If you really want the alignment and padding, they should go into a structure. ;-)
OK, so we would bump to liburcu2 then for the 0.8 release ?
I don't believe that this optimization by itself warrants a version bump. Perhaps hold it in reserve when the next need for a version bump comes along.
So, how to do this without a version bump? One approach would be to use linker-script magic (yeah, now -that- will be portable!). Or if the alignment attributes happen to do the right thing on popular gcc versions, go with them until the next version bump allows us to do it more deterministically.
Thanx, Paul
Quoted 188 lines
Thanks,
Mathieu
Quoted 177 lines
Thanx, Paul
Quoted 173 lines
Thanks,
Mathieu
Quoted 162 lines
Quoted 162 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 27 lines
Quoted 27 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 5 lines
Quoted 5 lines
urcu.c static void wait_for_readers(struct cds_list_head · line 282
Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 45 lines · Paul E. McKenney
Quoted 44 lines · Mathieu Desnoyers
Quoted 42 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 30 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
we can define
rcu_gp_ctrand registry with aligned attribute, but it isnot
Quoted 5 lines
reliable way
We need only this:
unsigned longrcu_gp_ctr__attribute((aligned and padded(don't put other var next to it except the futex)))In which situation would this be unreliable ?
Quoted 6 lines
int a;
int b __attribute__((aligned));
int c;b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line
So if our goal is to have
rcu_gp_ctrandrcu_gp_futexon the same cache line, which is different from that of the registry, we could do:typeA
rcu_gp_ctr__attribute__((aligned(...)));
typeBrcu_gp_futex;
typeC registry __attribute__((aligned(...)));Quoted 8 lines
I would expect the compiler won't typically reorder
rcu_gp_futexand registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.I guess this is a case where we could bump the library version number and do things properly.
Let's think a bit more about it, anyone else has comments on this ?
If you really want the alignment and padding, they should go into a structure. ;-)
Quoted 8 lines
Quoted 1 line
OK, so we would bump to liburcu2 then for the 0.8 release ?
I don't believe that this optimization by itself warrants a version bump. Perhaps hold it in reserve when the next need for a version bump comes along.
So, how to do this without a version bump? One approach would be to use linker-script magic
I already did it, send soon.
Quoted 197 lines
(yeah, now -that- will be portable!). Or if
the alignment attributes happen to do the right thing on popular gcc versions, go with them until the next version bump allows us to do it more deterministically.Thanx, Paul
Quoted 188 lines
Thanks,
Mathieu
Quoted 177 lines
Thanx, Paul
Quoted 173 lines
Thanks,
Mathieu
Quoted 162 lines
Quoted 162 lines
Thanks,
Mathieu
Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines
* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)
Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but they always be accessed at almost the same time, so we also move
rcu_gp_futexQuoted 1 line
to the cacheline of
rcu_gp_ctrto reduce the cacheline-usage orQuoted 8 lines
Quoted 8 lines
cache-missing
Quoted 1 line
of read site.
Hi Lai,
I agree on the goal: placing registry and
rcu_gp_ctron different cache-lines. And yes, it makes sense to putrcu_gp_ctrandrcu_gp_futexon the same cache-line. I agree that the next patch is fine too(keeping
Quoted 7 lines
Quoted 7 lines
qsbr and other urcu similar). This is indeed what I try to ensure myself.
I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or ifwe
Quoted 10 lines
Quoted 10 lines
choose to bump the library version number.
Thoughts ?
Thanks,
Mathieu
Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads2100285330
nr_writesQuoted 4 lines
Quoted 4 lines
3390219 nr_ops 2103675549Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1619868562
nr_writesQuoted 4 lines
Quoted 4 lines
3529478 nr_ops 1623398040Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads1949067038
nr_writesQuoted 8 lines
Quoted 8 lines
3469334 nr_ops 1952536372Quoted 6 lines
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3380191848
nr_writesQuoted 4 lines
Quoted 4 lines
4903248 nr_ops 3385095096Quoted 2 lines
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur0 wdur 0
nr_writers1 wdelay 0nr_reads3397637486
nr_writesQuoted 11 lines
Quoted 11 lines
4129809 nr_ops 3401767295Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>---
urcu.c +0
-0@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif Quoted 9 lines
Quoted 9 lines
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93
Quoted 9 lines
urcu.c +0
-8static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and
Quoted 7 lines
Quoted 7 lines
readers.
urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89
Quoted 6 lines
urcu.c +1
-2-*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader
Quoted 12 lines
Quoted 12 lines
and the
Quoted 11 lines
urcu.c +2
-2@@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }Quoted 1 line
Quoted 1 line
urcu.c static void wait_for_readers(struct
Quoted 1 line
urcu.c +0
-0@@ -232,12 +223,12 @@ static void wait_for_readers(struct
cds_list_headQuoted 27 lines
Quoted 27 lines
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 15 lines
urcu.c +2
-2/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,urcu.c static void wait_for_readers(struct cds_list_head
Quoted 9 lines
urcu.c +1
-1if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head
*input_readers,Quoted 5 lines
Quoted 5 lines
urcu.c static void wait_for_readers(struct cds_list_head · line 282
Quoted 3 lines
urcu.c +0
-0if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ > > [email protected]<javascript:;><javascript:;>--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.comSignature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
PATCH v2 1/1 urcu: avoid false sharing for rcu_gp_ctr
In reply to a message not in the archiveFrom 66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90 Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing for rcu_gp_ctr
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.
test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372
after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295
--- Makefile.am | 10 +++++----- compat_rcu_gp.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_bp.lds | 24 ++++++++++++++++++++++++ compat_rcu_gp_mb.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_qsbr.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_signal.lds | 25 +++++++++++++++++++++++++ urcu-bp.c | 14 ++++---------- urcu-qsbr.c | 35 +++++++++++++++-------------------- urcu.c | 34 +++++++++++++--------------------- urcu/map/urcu-bp.h | 3 +-- urcu/map/urcu-qsbr.h | 3 +-- urcu/map/urcu.h | 9 +++------ urcu/static/urcu-bp.h | 23 ++++++++++++++--------- urcu/static/urcu-qsbr.h | 39 ++++++++++++++++++++++++--------------- urcu/static/urcu.h | 38 ++++++++++++++++++++++++-------------- 15 files changed, 228 insertions(+), 104 deletions(-) create mode 100644 compat_rcu_gp.lds create mode 100644 compat_rcu_gp_bp.lds create mode 100644 compat_rcu_gp_mb.lds create mode 100644 compat_rcu_gp_qsbr.lds create mode 100644 compat_rcu_gp_signal.lds
Makefile.am +5 -5
@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \ # liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT) -liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT) liburcu_la_LIBADD = liburcu-common.la -liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT) +liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT) liburcu_qsbr_la_LIBADD = liburcu-common.la -liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT) liburcu_mb_la_CFLAGS = -DRCU_MB liburcu_mb_la_LIBADD = liburcu-common.la -liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_signal.lds $(COMPAT) liburcu_signal_la_CFLAGS = -DRCU_SIGNAL liburcu_signal_la_LIBADD = liburcu-common.la -liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT) +liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c compat_rcu_gp_bp.lds $(COMPAT) liburcu_bp_la_LIBADD = liburcu-common.la liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \
compat_rcu_gp.lds new file +25 -0
@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp.lds + * + * Linker Script for compatibility of liburcu + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_memb = rcu_gp_memb; +rcu_gp_futex_memb = rcu_gp_memb + 8; +
compat_rcu_gp_bp.lds new file +24 -0
@@ -0,0 +1,24 @@ +/* + * compat_rcu_gp_bp.lds + * + * Linker Script for compatibility of liburcu-bp + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_bp = rcu_gp_bp; +
compat_rcu_gp_mb.lds new file +25 -0
@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_mb.lds + * + * Linker Script for compatibility of liburcu-mb + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_mb = rcu_gp_mb; +rcu_gp_futex_mb = rcu_gp_mb + 8; +
compat_rcu_gp_qsbr.lds new file +25 -0
@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_qsbr.lds + * + * Linker Script for compatibility of liburcu-qsbr + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_qsbr = rcu_gp_qsbr; +rcu_gp_futex_qsbr = rcu_gp_qsbr + 8; +
compat_rcu_gp_signal.lds new file +25 -0
@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_signal.lds + * + * Linker Script for compatibility of liburcu-signal + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_signal = rcu_gp_signal; +rcu_gp_futex_signal = rcu_gp_signal + 8; +
urcu-bp.c +4 -10
@@ -103,6 +103,8 @@ void *mremap_wrapper(void *old_address, size_t old_size,void __attribute__((destructor))rcu_bp_exit (void );static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; +/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };#ifdef DEBUG_YIELD unsigned int rcu_yield_active;
@@ -110,14 +112,6 @@ DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);#endif /* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; - -/* * Pointer to registry elements. Written to only by each individual reader. Read * by both the reader and the writers. */
@@ -174,7 +168,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;
@@ -250,7 +244,7 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* * Must commit qparity update to memory before waiting for other parity
urcu-qsbr.c +15 -20
@@ -53,13 +53,8 @@void __attribute__((destructor))rcu_exit (void );static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -*/ -unsigned long rcu_gp_ctr =RCU_GP_ONLINE ; +/* Init the ctr as RCU_GP_ONLINE, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { { .ctr =RCU_GP_ONLINE } };/* * Active attempts to check for reader Q.S. before calling futex().
@@ -118,8 +113,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ cmm_smp_rmb (); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_noasync (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_noasync (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }
@@ -133,12 +128,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (offline), or for them to observe the -* current rcu_gp_ctr value. +* current rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops >=RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_set (&rcu_gp_futex,-1 ); +uatomic_set (&rcu_gp.futex,-1 );/* * Write futex before write waiting (the other side * reads them in the opposite order).
@@ -177,7 +172,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops >=RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ cmm_smp_mb (); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {
@@ -253,11 +248,11 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity -* before committing next rcu_gp_ctr update to memory. Failure +* before committing next rcu_gp.ctr update to memory. Failure * to do so could result in the writer waiting forever while new * readers are always accessing data (no progress). Enforce * compiler-order of load URCU_TLS(rcu_reader).ctr before store -* to rcu_gp_ctr. +* to rcu_gp.ctr. */ cmm_barrier ();
@@ -269,13 +264,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR );/* -* Must commit rcu_gp_ctr update to memory before waiting for +* Must commit rcu_gp.ctr update to memory before waiting for * quiescent state. Failure to do so could result in the writer * waiting forever while new readers are always accessing data -* (no progress). Enforce compiler-order of store to rcu_gp_ctr +* (no progress). Enforce compiler-order of store to rcu_gp.ctr * before load URCU_TLS(rcu_reader).ctr. */ cmm_barrier ();
@@ -353,13 +348,13 @@ void synchronize_rcu(void) goto out;/* Increment current G.P. */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr +RCU_GP_CTR ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr +RCU_GP_CTR );/* -* Must commit rcu_gp_ctr update to memory before waiting for +* Must commit rcu_gp.ctr update to memory before waiting for * quiescent state. Failure to do so could result in the writer * waiting forever while new readers are always accessing data -* (no progress). Enforce compiler-order of store to rcu_gp_ctr +* (no progress). Enforce compiler-order of store to rcu_gp.ctr * before load URCU_TLS(rcu_reader).ctr. */ cmm_barrier ();
urcu.c +13 -21
@@ -83,16 +83,8 @@ void __attribute__((destructor)) rcu_exit(void);#endif static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { { .ctr =RCU_GP_COUNT } };/* * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +209,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }
@@ -232,12 +224,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }
@@ -270,7 +262,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {
@@ -289,7 +281,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {
@@ -357,10 +349,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before -* committing next rcu_gp_ctr update to memory. Failure to do so could +* committing next rcu_gp.ctr update to memory. Failure to do so could * result in the writer waiting forever while new readers are always * accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();
@@ -372,13 +364,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent +* Must commit rcu_gp.ctr update to memory before waiting for quiescent * state. Failure to do so could result in the writer waiting forever * while new readers are always accessing data (no progress). Enforce -* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr. +* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr. */ cmm_barrier ();
urcu/map/urcu-bp.h +1 -2
@@ -44,8 +44,7 @@#define rcu_exit rcu_exit_bp#define synchronize_rcu synchronize_rcu_bp#define rcu_reader rcu_reader_bp -#define rcu_gp_ctr rcu_gp_ctr_bp -#define rcu_gp_futex rcu_gp_futex_bp/* unused */ +#define rcu_gp rcu_gp_bp#define get_cpu_call_rcu_data get_cpu_call_rcu_data_bp#define get_call_rcu_thread get_call_rcu_thread_bp
urcu/map/urcu-qsbr.h +1 -2
@@ -47,8 +47,7 @@#define rcu_exit rcu_exit_qsbr#define synchronize_rcu synchronize_rcu_qsbr#define rcu_reader rcu_reader_qsbr -#define rcu_gp_ctr rcu_gp_ctr_qsbr -#define rcu_gp_futex rcu_gp_futex_qsbr +#define rcu_gp rcu_gp_qsbr#define get_cpu_call_rcu_data get_cpu_call_rcu_data_qsbr#define get_call_rcu_thread get_call_rcu_thread_qsbr
urcu/map/urcu.h +3 -6
@@ -75,8 +75,7 @@#define rcu_exit rcu_exit_memb#define synchronize_rcu synchronize_rcu_memb#define rcu_reader rcu_reader_memb -#define rcu_gp_ctr rcu_gp_ctr_memb -#define rcu_gp_futex rcu_gp_futex_memb +#define rcu_gp rcu_gp_memb#define get_cpu_call_rcu_data get_cpu_call_rcu_data_memb#define get_call_rcu_thread get_call_rcu_thread_memb
@@ -121,8 +120,7 @@#define rcu_exit rcu_exit_sig#define synchronize_rcu synchronize_rcu_sig#define rcu_reader rcu_reader_sig -#define rcu_gp_ctr rcu_gp_ctr_sig -#define rcu_gp_futex rcu_gp_futex_sig +#define rcu_gp rcu_gp_sig#define get_cpu_call_rcu_data get_cpu_call_rcu_data_sig#define get_call_rcu_thread get_call_rcu_thread_sig
@@ -164,8 +162,7 @@#define rcu_exit rcu_exit_mb#define synchronize_rcu synchronize_rcu_mb#define rcu_reader rcu_reader_mb -#define rcu_gp_ctr rcu_gp_ctr_mb -#define rcu_gp_futex rcu_gp_futex_mb +#define rcu_gp rcu_gp_mb#define get_cpu_call_rcu_data get_cpu_call_rcu_data_mb#define get_call_rcu_thread get_call_rcu_thread_mb
urcu/static/urcu-bp.h +14 -9
@@ -130,12 +130,17 @@ static inline void rcu_debug_yield_init(void)*/ extern void rcu_bp_register (void ); -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global quiescent period counter with low-order bits unused. +* Contains the current RCU_GP_CTR_PHASE. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */
@@ -166,13 +171,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
@@ -181,7 +186,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));cmm_smp_mb (); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr, tmp +RCU_GP_COUNT );
urcu/static/urcu-qsbr.h +24 -15
@@ -119,12 +119,23 @@ static inline void rcu_debug_yield_init(void)#define RCU_GP_ONLINE (1UL << 0)#define RCU_GP_CTR (1UL << 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +union { +/* +* Global quiescent period counter with low-order bits unused. +* Using a int rather than a char to eliminate false register +* dependencies causing stalls on some architectures. +*/ +unsigned long ctr; + +/* ensure the futex is located at offset=8 */ +int64_t __unused; + }; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */
@@ -137,8 +148,6 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */
@@ -147,10 +156,10 @@ static inline void wake_up_gp(void)if (caa_unlikely (_CMM_LOAD_SHARED (URCU_TLS (rcu_reader).waiting))) {_CMM_STORE_SHARED (URCU_TLS (rcu_reader).waiting,0 );cmm_smp_mb (); -if (uatomic_read (&rcu_gp_futex) !=-1 ) +if (uatomic_read (&rcu_gp.futex) !=-1 )return ; -uatomic_set (&rcu_gp_futex,0 ); -futex_noasync (&rcu_gp_futex,FUTEX_WAKE ,1 , +uatomic_set (&rcu_gp.futex,0 ); +futex_noasync (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }
@@ -162,7 +171,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!v)return RCU_READER_INACTIVE ; -if (v == rcu_gp_ctr) +if (v == rcu_gp.ctr)return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }
@@ -216,7 +225,7 @@ static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)* to be invoked directly from non-LGPL code. * * We skip the memory barriers and gp store if our local ctr already -* matches the global rcu_gp_ctr value: this is OK because a prior +* matches the global rcu_gp.ctr value: this is OK because a prior * _rcu_quiescent_state() or _rcu_thread_online() already updated it * within our thread, so we have no quiescent state to report. */
@@ -224,7 +233,7 @@ static inline void _rcu_quiescent_state(void) {unsigned long gp_ctr; -if ((gp_ctr =CMM_LOAD_SHARED (rcu_gp_ctr)) ==URCU_TLS (rcu_reader).ctr) +if ((gp_ctr =CMM_LOAD_SHARED (rcu_gp.ctr)) ==URCU_TLS (rcu_reader).ctr)return ;_rcu_quiescent_state_update_and_wakeup (gp_ctr); }
@@ -257,7 +266,7 @@ static inline void _rcu_thread_offline(void)static inline void _rcu_thread_online (void ) {cmm_barrier ();/* Ensure the compiler does not reorder us with mutex */ -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,CMM_LOAD_SHARED (rcu_gp.ctr));cmm_smp_mb (); }
urcu/static/urcu.h +24 -14
@@ -213,12 +213,24 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +union { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +/* ensure the futex is located at offset=8 */ +int64_t __unused; + }; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +243,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }
@@ -256,13 +266,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
@@ -271,7 +281,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr, tmp +RCU_GP_COUNT );
Signature · 1 line
1.7.4.4
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 62 lines · * Lai Jiangshan ([email protected]) wrote:
From
66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing forrcu_gp_ctr@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but
they always be accessed at almost the same time, so we also movercu_gp_futexto the cacheline ofrcu_gp_ctrto reduce the cacheline-usage or cache-missing of read site.test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219nr_ops2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478nr_ops1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334nr_ops1952536372after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248nr_ops3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809nr_ops3401767295Signed-off-by Lai Jiangshan <[email protected]>--- Makefile.am | 10 +++++----- compat_rcu_gp.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_bp.lds | 24 ++++++++++++++++++++++++ compat_rcu_gp_mb.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_qsbr.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_signal.lds | 25 +++++++++++++++++++++++++ urcu-bp.c | 14 ++++---------- urcu-qsbr.c | 35 +++++++++++++++-------------------- urcu.c | 34 +++++++++++++--------------------- urcu/map/urcu-bp.h | 3 +-- urcu/map/urcu-qsbr.h | 3 +-- urcu/map/urcu.h | 9 +++------ urcu/static/urcu-bp.h | 23 ++++++++++++++--------- urcu/static/urcu-qsbr.h | 39 ++++++++++++++++++++++++--------------- urcu/static/urcu.h | 38 ++++++++++++++++++++++++-------------- 15 files changed, 228 insertions(+), 104 deletions(-) create mode 100644 compat_rcu_gp.lds create mode 100644 compat_rcu_gp_bp.lds create mode 100644 compat_rcu_gp_mb.lds create mode 100644 compat_rcu_gp_qsbr.lds create mode 100644 compat_rcu_gp_signal.ldsMakefile.am +0
-0
Makefile.am lib_LTLIBRARIES = liburcu-common.la \ · line 59
Quoted 5 lines
Makefile.am +1
-1@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \ # liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT) -liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT)
Hrm, I tried your patch with "make V=1", and have not found any use of those "lds" files. I'm not sure adding them as SOURCES is enough.
Quoted 550 lines
Makefile.am +4
-4liburcu_la_LIBADD = liburcu-common.la -liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT) +liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT) liburcu_qsbr_la_LIBADD = liburcu-common.la -liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT) liburcu_mb_la_CFLAGS = -DRCU_MB liburcu_mb_la_LIBADD = liburcu-common.la -liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_signal.lds $(COMPAT) liburcu_signal_la_CFLAGS = -DRCU_SIGNAL liburcu_signal_la_LIBADD = liburcu-common.la -liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT) +liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c compat_rcu_gp_bp.lds $(COMPAT) liburcu_bp_la_LIBADD = liburcu-common.la liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \compat_rcu_gp.lds new file +25
-0@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp.lds + * + * Linker Script for compatibility of liburcu + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_memb = rcu_gp_memb; +rcu_gp_futex_memb = rcu_gp_memb + 8; +compat_rcu_gp_bp.lds new file +24
-0@@ -0,0 +1,24 @@ +/* + * compat_rcu_gp_bp.lds + * + * Linker Script for compatibility of liburcu-bp + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_bp = rcu_gp_bp; +compat_rcu_gp_mb.lds new file +25
-0@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_mb.lds + * + * Linker Script for compatibility of liburcu-mb + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_mb = rcu_gp_mb; +rcu_gp_futex_mb = rcu_gp_mb + 8; +compat_rcu_gp_qsbr.lds new file +25
-0@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_qsbr.lds + * + * Linker Script for compatibility of liburcu-qsbr + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_qsbr = rcu_gp_qsbr; +rcu_gp_futex_qsbr = rcu_gp_qsbr + 8; +compat_rcu_gp_signal.lds new file +25
-0@@ -0,0 +1,25 @@ +/* + * compat_rcu_gp_signal.lds + * + * Linker Script for compatibility of liburcu-signal + * + * Copyright (c) 2012 Lai Jiangshan <[email protected]> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +rcu_gp_ctr_signal = rcu_gp_signal; +rcu_gp_futex_signal = rcu_gp_signal + 8; +urcu-bp.c +4
-10@@ -103,6 +103,8 @@ void *mremap_wrapper(void *old_address, size_t old_size,void __attribute__((destructor))rcu_bp_exit (void );static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; +/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };#ifdef DEBUG_YIELD unsigned int rcu_yield_active;@@ -110,14 +112,6 @@ DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);#endif /* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; - -/* * Pointer to registry elements. Written to only by each individual reader. Read * by both the reader and the writers. */ @@ -174,7 +168,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;@@ -250,7 +244,7 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* * Must commit qparity update to memory before waiting for other parity urcu-qsbr.c +15
-20@@ -53,13 +53,8 @@void __attribute__((destructor))rcu_exit (void );static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -*/ -unsigned long rcu_gp_ctr =RCU_GP_ONLINE ; +/* Init the ctr as RCU_GP_ONLINE, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { { .ctr =RCU_GP_ONLINE } };/* * Active attempts to check for reader Q.S. before calling futex(). @@ -118,8 +113,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ cmm_smp_rmb (); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_noasync (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_noasync (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -133,12 +128,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (offline), or for them to observe the -* current rcu_gp_ctr value. +* current rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops >=RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_set (&rcu_gp_futex,-1 ); +uatomic_set (&rcu_gp.futex,-1 );/* * Write futex before write waiting (the other side * reads them in the opposite order). @@ -177,7 +172,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops >=RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ cmm_smp_mb (); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -253,11 +248,11 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity -* before committing next rcu_gp_ctr update to memory. Failure +* before committing next rcu_gp.ctr update to memory. Failure * to do so could result in the writer waiting forever while new * readers are always accessing data (no progress). Enforce * compiler-order of load URCU_TLS(rcu_reader).ctr before store -* to rcu_gp_ctr. +* to rcu_gp.ctr. */ cmm_barrier ();@@ -269,13 +264,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR );/* -* Must commit rcu_gp_ctr update to memory before waiting for +* Must commit rcu_gp.ctr update to memory before waiting for * quiescent state. Failure to do so could result in the writer * waiting forever while new readers are always accessing data -* (no progress). Enforce compiler-order of store to rcu_gp_ctr +* (no progress). Enforce compiler-order of store to rcu_gp.ctr * before load URCU_TLS(rcu_reader).ctr. */ cmm_barrier ();@@ -353,13 +348,13 @@ void synchronize_rcu(void) goto out;/* Increment current G.P. */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr +RCU_GP_CTR ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr +RCU_GP_CTR );/* -* Must commit rcu_gp_ctr update to memory before waiting for +* Must commit rcu_gp.ctr update to memory before waiting for * quiescent state. Failure to do so could result in the writer * waiting forever while new readers are always accessing data -* (no progress). Enforce compiler-order of store to rcu_gp_ctr +* (no progress). Enforce compiler-order of store to rcu_gp.ctr * before load URCU_TLS(rcu_reader).ctr. */ cmm_barrier ();urcu.c +13
-21@@ -83,16 +83,8 @@ void __attribute__((destructor)) rcu_exit(void);#endif static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */ +struct urcu_gp rcu_gp = { { .ctr =RCU_GP_COUNT } };/* * Written to only by each individual reader. Read by both the reader and the @@ -217,8 +209,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +224,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +262,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +281,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +349,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before -* committing next rcu_gp_ctr update to memory. Failure to do so could +* committing next rcu_gp.ctr update to memory. Failure to do so could * result in the writer waiting forever while new readers are always * accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();@@ -372,13 +364,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent +* Must commit rcu_gp.ctr update to memory before waiting for quiescent * state. Failure to do so could result in the writer waiting forever * while new readers are always accessing data (no progress). Enforce -* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr. +* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr. */ cmm_barrier ();urcu/map/urcu-bp.h +1
-2@@ -44,8 +44,7 @@#define rcu_exit rcu_exit_bp#define synchronize_rcu synchronize_rcu_bp#define rcu_reader rcu_reader_bp -#define rcu_gp_ctr rcu_gp_ctr_bp -#define rcu_gp_futex rcu_gp_futex_bp/* unused */ +#define rcu_gp rcu_gp_bp#define get_cpu_call_rcu_data get_cpu_call_rcu_data_bp#define get_call_rcu_thread get_call_rcu_thread_bpurcu/map/urcu-qsbr.h +1
-2@@ -47,8 +47,7 @@#define rcu_exit rcu_exit_qsbr#define synchronize_rcu synchronize_rcu_qsbr#define rcu_reader rcu_reader_qsbr -#define rcu_gp_ctr rcu_gp_ctr_qsbr -#define rcu_gp_futex rcu_gp_futex_qsbr +#define rcu_gp rcu_gp_qsbr#define get_cpu_call_rcu_data get_cpu_call_rcu_data_qsbr#define get_call_rcu_thread get_call_rcu_thread_qsbrurcu/map/urcu.h +3
-6@@ -75,8 +75,7 @@#define rcu_exit rcu_exit_memb#define synchronize_rcu synchronize_rcu_memb#define rcu_reader rcu_reader_memb -#define rcu_gp_ctr rcu_gp_ctr_memb -#define rcu_gp_futex rcu_gp_futex_memb +#define rcu_gp rcu_gp_memb#define get_cpu_call_rcu_data get_cpu_call_rcu_data_memb#define get_call_rcu_thread get_call_rcu_thread_memb@@ -121,8 +120,7 @@#define rcu_exit rcu_exit_sig#define synchronize_rcu synchronize_rcu_sig#define rcu_reader rcu_reader_sig -#define rcu_gp_ctr rcu_gp_ctr_sig -#define rcu_gp_futex rcu_gp_futex_sig +#define rcu_gp rcu_gp_sig#define get_cpu_call_rcu_data get_cpu_call_rcu_data_sig#define get_call_rcu_thread get_call_rcu_thread_sig@@ -164,8 +162,7 @@#define rcu_exit rcu_exit_mb#define synchronize_rcu synchronize_rcu_mb#define rcu_reader rcu_reader_mb -#define rcu_gp_ctr rcu_gp_ctr_mb -#define rcu_gp_futex rcu_gp_futex_mb +#define rcu_gp rcu_gp_mb#define get_cpu_call_rcu_data get_cpu_call_rcu_data_mb#define get_call_rcu_thread get_call_rcu_thread_mburcu/static/urcu-bp.h +14
-9@@ -130,12 +130,17 @@ static inline void rcu_debug_yield_init(void)*/ extern void rcu_bp_register (void ); -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global quiescent period counter with low-order bits unused. +* Contains the current RCU_GP_CTR_PHASE. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -166,13 +171,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -181,7 +186,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));cmm_smp_mb (); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader)->ctr, tmp +RCU_GP_COUNT );urcu/static/urcu-qsbr.h +0
-6@@ -119,12 +119,23 @@ static inline void rcu_debug_yield_init(void)#define RCU_GP_ONLINE (1UL << 0)#define RCU_GP_CTR (1UL << 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr;
urcu/static/urcu-qsbr.h static inline void rcu_debug_yield_init(void) · line 131
Quoted 10 lines
urcu/static/urcu-qsbr.h +10
-0+struct urcu_gp { +union { +/* +* Global quiescent period counter with low-order bits unused. +* Using a int rather than a char to eliminate false register +* dependencies causing stalls on some architectures. +*/ +unsigned long ctr; + +/* ensure the futex is located at offset=8 */
The comment could probably be more verbose:
/*
* Ensure futex field is located at offset=8 for backward
* compatibility linker script.
*/
urcu/static/urcu-qsbr.h static inline void rcu_debug_yield_init(void) · line 132
Quoted 1 line
urcu/static/urcu-qsbr.h +1
-0+int64_t __unused;
I would be tempted to do:
urcu-linker.h:
#define URCU_GP_FUTEX_OFFSET 8
and use a
char __unused[URCU_GP_FUTEX_OFFSET];
in the structure (and use this macro in the linker scripts as well).
The advantage of using a char rather than a int64_t is that it won't get in the way of type pruning, and will allow more aliasing analysis to take place: AFAIK, adding a int64_t aliased with a unsigned long within a union requires that a compiler doing type pruning considers that those two types can alias pretty much anywhere in a file including our header.
Thanks,
Mathieu
Quoted 151 lines
urcu/static/urcu-qsbr.h +13
-9+ }; + +int32_t futex; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -137,8 +148,6 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ @@ -147,10 +156,10 @@ static inline void wake_up_gp(void)if (caa_unlikely (_CMM_LOAD_SHARED (URCU_TLS (rcu_reader).waiting))) {_CMM_STORE_SHARED (URCU_TLS (rcu_reader).waiting,0 );cmm_smp_mb (); -if (uatomic_read (&rcu_gp_futex) !=-1 ) +if (uatomic_read (&rcu_gp.futex) !=-1 )return ; -uatomic_set (&rcu_gp_futex,0 ); -futex_noasync (&rcu_gp_futex,FUTEX_WAKE ,1 , +uatomic_set (&rcu_gp.futex,0 ); +futex_noasync (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -162,7 +171,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!v)return RCU_READER_INACTIVE ; -if (v == rcu_gp_ctr) +if (v == rcu_gp.ctr)return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }@@ -216,7 +225,7 @@ static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)* to be invoked directly from non-LGPL code. * * We skip the memory barriers and gp store if our local ctr already -* matches the global rcu_gp_ctr value: this is OK because a prior +* matches the global rcu_gp.ctr value: this is OK because a prior * _rcu_quiescent_state() or _rcu_thread_online() already updated it * within our thread, so we have no quiescent state to report. */ @@ -224,7 +233,7 @@ static inline void _rcu_quiescent_state(void) {unsigned long gp_ctr; -if ((gp_ctr =CMM_LOAD_SHARED (rcu_gp_ctr)) ==URCU_TLS (rcu_reader).ctr) +if ((gp_ctr =CMM_LOAD_SHARED (rcu_gp.ctr)) ==URCU_TLS (rcu_reader).ctr)return ;_rcu_quiescent_state_update_and_wakeup (gp_ctr); }@@ -257,7 +266,7 @@ static inline void _rcu_thread_offline(void)static inline void _rcu_thread_online (void ) {cmm_barrier ();/* Ensure the compiler does not reorder us with mutex */ -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,CMM_LOAD_SHARED (rcu_gp.ctr));cmm_smp_mb (); }urcu/static/urcu.h +24
-14@@ -213,12 +213,24 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +union { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +/* ensure the futex is located at offset=8 */ +int64_t __unused; + }; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -231,16 +243,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +266,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +281,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr, tmp +RCU_GP_COUNT );Signature · 1 line
1.7.4.4
Signature · 4 lines
Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20121211123802.GA4356@Krystal
Quoted 66 lines · Mathieu Desnoyers
Quoted 65 lines · * Lai Jiangshan ([email protected]) wrote:
From
66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing forrcu_gp_ctr@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but
they always be accessed at almost the same time, so we also movercu_gp_futexto the cacheline ofrcu_gp_ctrto reduce the cacheline-usage or cache-missing of read site.test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219nr_ops2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478nr_ops1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334nr_ops1952536372after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248nr_ops3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809nr_ops3401767295Signed-off-by Lai Jiangshan <[email protected]>--- Makefile.am | 10 +++++----- compat_rcu_gp.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_bp.lds | 24 ++++++++++++++++++++++++ compat_rcu_gp_mb.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_qsbr.lds | 25 +++++++++++++++++++++++++ compat_rcu_gp_signal.lds | 25 +++++++++++++++++++++++++ urcu-bp.c | 14 ++++---------- urcu-qsbr.c | 35 +++++++++++++++-------------------- urcu.c | 34 +++++++++++++--------------------- urcu/map/urcu-bp.h | 3 +-- urcu/map/urcu-qsbr.h | 3 +-- urcu/map/urcu.h | 9 +++------ urcu/static/urcu-bp.h | 23 ++++++++++++++--------- urcu/static/urcu-qsbr.h | 39 ++++++++++++++++++++++++--------------- urcu/static/urcu.h | 38 ++++++++++++++++++++++++-------------- 15 files changed, 228 insertions(+), 104 deletions(-) create mode 100644 compat_rcu_gp.lds create mode 100644 compat_rcu_gp_bp.lds create mode 100644 compat_rcu_gp_mb.lds create mode 100644 compat_rcu_gp_qsbr.lds create mode 100644 compat_rcu_gp_signal.ldsMakefile.am +0
-0@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \ # liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
Quoted 5 lines
Makefile.am lib_LTLIBRARIES = liburcu-common.la \ · line 59
Quoted 2 lines
Makefile.am +1
-1-liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT)Hrm, I tried your patch with "make V=1", and have not found any use of those "lds" files. I'm not sure adding them as SOURCES is enough.
Use liburcu_mb_la_LDFLAGS = -Xlinker compat_rcu_gp_mb.lds instead!
Result(nm .libs/liburcu-mb.so):00000000002046e0 D rcu_flavor_mb0000000000204a80 D rcu_gp_ctr_mb0000000000204a88 A rcu_gp_futex_mb0000000000204c60 b rcu_gp_lock0000000000204a80 D rcu_gp_mb
The type "A" makes me nervous, Do we expect this?
And a new problem:
liburcu.so is RCU_MB actually in my system, but compat_rcu_gp.lds uses rcu_gp_memb(RCU_MEMBARRIER), it can't work in my system, how to fix it?
Quoted 11 lines
Quoted 11 lines
Makefile.am +2
-2liburcu_la_LIBADD = liburcu-common.la -liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT) +liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT) liburcu_qsbr_la_LIBADD = liburcu-common.la -liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) +liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT) liburcu_mb_la_CFLAGS = -DRCU_MB liburcu_mb_la_LIBADD = liburcu-common.la
Since we are bumping the urcu soname version number to 2.0.0 for the upcoming urcu 0.8 anyway, it's time to merge this patch.
Thanks!
Mathieu
Quoted 204 lines · * Lai Jiangshan ([email protected]) wrote:
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but
they always be accessed at almost the same time, so we also movercu_gp_futexto the cacheline ofrcu_gp_ctrto reduce the cacheline-usage or cache-missing of read site.test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219nr_ops2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478nr_ops1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334nr_ops1952536372after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248nr_ops3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809nr_ops3401767295Singed-by Lai Jiangshan <[email protected]>---
urcu.c +12
-21@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader and the @@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +348,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before -* committing next rcu_gp_ctr update to memory. Failure to do so could +* committing next rcu_gp.ctr update to memory. Failure to do so could * result in the writer waiting forever while new readers are always * accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();@@ -372,13 +363,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent +* Must commit rcu_gp.ctr update to memory before waiting for quiescent * state. Failure to do so could result in the writer waiting forever * while new readers are always accessing data (no progress). Enforce -* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr. +* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr. */ cmm_barrier ();urcu/static/urcu.h +20
-14@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -231,16 +239,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr, tmp +RCU_GP_COUNT );
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20130506124433.GB12699@Krystal
Hi, Mathieu,
There is a big compatible problem in URCU which should be fix in next round.
LB: liburcu built on the system which has sys_membarrier().
LU: liburcu built on the system which does NOT have sys_membarrier().
LBM: liburcu-mb ....
LUM: liburcu-mb ...
AB: application(-lliburcu) built on the system which has sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT have sys_membarrier().
ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...
AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinations
I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?
could you check it?
How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to URCU_MB.
Thanks,
Lai
Quoted 213 lines · Mathieu Desnoyers
Since we are bumping the urcu soname version number to 2.0.0 for the upcoming urcu 0.8 anyway, it's time to merge this patch.
Thanks!
Mathieu
Quoted 204 lines · * Lai Jiangshan ([email protected]) wrote:
@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.
Fix: Use different cache line for them.
Although
rcu_gp_futexis updated less thanrcu_gp_ctr, but
they always be accessed at almost the same time, so we also movercu_gp_futexto the cacheline ofrcu_gp_ctrto reduce the cacheline-usage or cache-missing of read site.test: (4X6=24 CPUs)
Before patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads2100285330nr_writes3390219nr_ops2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1619868562nr_writes3529478nr_ops1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads1949067038nr_writes3469334nr_ops1952536372after patch:
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3380191848nr_writes4903248nr_ops3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20nr_readers20 rdur 0 wdur 0nr_writers1 wdelay 0nr_reads3397637486nr_writes4129809nr_ops3401767295Singed-by Lai Jiangshan <[email protected]>---
urcu.c +12
-21@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);#endif static pthread_mutex_t rcu_gp_lock =PTHREAD_MUTEX_INITIALIZER ; - -int32_t rcu_gp_futex; - -/* -* Global grace period counter. -* Contains the current RCU_GP_CTR_PHASE. -* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. -* Written to only by writer with mutex taken. Read by both writer and readers. -*/ -unsigned long rcu_gp_ctr =RCU_GP_COUNT ; +struct urcu_gp rcu_gp = { .ctr =RCU_GP_COUNT };/* * Written to only by each individual reader. Read by both the reader and the @@ -217,8 +208,8 @@ static void wait_gp(void) {/* Read reader_gp before read futex */ smp_mb_master (RCU_MB_GROUP ); -if (uatomic_read (&rcu_gp_futex) ==-1 ) -futex_async (&rcu_gp_futex,FUTEX_WAIT ,-1 , +if (uatomic_read (&rcu_gp.futex) ==-1 ) +futex_async (&rcu_gp.futex,FUTEX_WAIT ,-1 ,NULL ,NULL ,0 ); }@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,/* * Wait for each thread URCU_TLS(rcu_reader).ctr to either * indicate quiescence (not nested), or observe the current -* rcu_gp_ctr value. +* rcu_gp.ctr value. */ for (;;) { wait_loops++;if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) { -uatomic_dec (&rcu_gp_futex); +uatomic_dec (&rcu_gp.futex);/* Write futex before read reader_gp */ smp_mb_master (RCU_MB_GROUP ); }@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,if (wait_loops ==RCU_QS_ACTIVE_ATTEMPTS ) {/* Read reader_gp before write futex */ smp_mb_master (RCU_MB_GROUP ); -uatomic_set (&rcu_gp_futex,0 ); +uatomic_set (&rcu_gp.futex,0 ); }break ; }else {@@ -357,10 +348,10 @@ void synchronize_rcu(void)/* * Must finish waiting for quiescent state for original parity before -* committing next rcu_gp_ctr update to memory. Failure to do so could +* committing next rcu_gp.ctr update to memory. Failure to do so could * result in the writer waiting forever while new readers are always * accessing data (no progress). Enforce compiler-order of load -* URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr. +* URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr. */ cmm_barrier ();@@ -372,13 +363,13 @@ void synchronize_rcu(void)cmm_smp_mb ();/* Switch parity: 0 -> 1, 1 -> 0 */ -CMM_STORE_SHARED (rcu_gp_ctr, rcu_gp_ctr ^RCU_GP_CTR_PHASE ); +CMM_STORE_SHARED (rcu_gp.ctr, rcu_gp.ctr ^RCU_GP_CTR_PHASE );/* -* Must commit rcu_gp_ctr update to memory before waiting for quiescent +* Must commit rcu_gp.ctr update to memory before waiting for quiescent * state. Failure to do so could result in the writer waiting forever * while new readers are always accessing data (no progress). Enforce -* compiler-order of store to rcu_gp_ctr before load rcu_reader ctr. +* compiler-order of store to rcu_gp.ctr before load rcu_reader ctr. */ cmm_barrier ();urcu/static/urcu.h +20
-14@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) -/* -* Global quiescent period counter with low-order bits unused. -* Using a int rather than a char to eliminate false register dependencies -* causing stalls on some architectures. -*/ -extern unsigned long rcu_gp_ctr; +struct urcu_gp { +/* +* Global grace period counter. +* Contains the current RCU_GP_CTR_PHASE. +* Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path. +* Written to only by writer with mutex taken. +* Read by both writer and readers. +*/ +unsigned long ctr; + +int32_t futex; +} __attribute__((aligned (CAA_CACHE_LINE_SIZE ))); + +extern struct urcu_gp rcu_gp;struct rcu_reader {/* Data used by both reader and synchronize_rcu() */ @@ -231,16 +239,14 @@ struct rcu_reader { extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader); -extern int32_t rcu_gp_futex; -/* * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. */ static inline void wake_up_gp (void ) { -if (caa_unlikely (uatomic_read (&rcu_gp_futex) ==-1 )) { -uatomic_set (&rcu_gp_futex,0 ); -futex_async (&rcu_gp_futex,FUTEX_WAKE ,1 , +if (caa_unlikely (uatomic_read (&rcu_gp.futex) ==-1 )) { +uatomic_set (&rcu_gp.futex,0 ); +futex_async (&rcu_gp.futex,FUTEX_WAKE ,1 ,NULL ,NULL ,0 ); } }@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr) v =CMM_LOAD_SHARED (*ctr);if (!(v &RCU_GP_CTR_NEST_MASK ))return RCU_READER_INACTIVE ; -if (!((v ^ rcu_gp_ctr) &RCU_GP_CTR_PHASE )) +if (!((v ^ rcu_gp.ctr) &RCU_GP_CTR_PHASE ))return RCU_READER_ACTIVE_CURRENT ;return RCU_READER_ACTIVE_OLD ; }/* -* Helper for _rcu_read_lock(). The format of rcu_gp_ctr (as well as +* Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as * the per-thread rcu_reader.ctr) has the upper bits containing a count of * _rcu_read_lock() nesting, and a lower-order bit that contains either zero * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in @@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)static inline void _rcu_read_lock_update (unsigned long tmp) {if (caa_likely (!(tmp &RCU_GP_CTR_NEST_MASK ))) { -_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp_ctr)); +_CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr,_CMM_LOAD_SHARED (rcu_gp.ctr));smp_mb_slave (RCU_MB_GROUP ); }else _CMM_STORE_SHARED (URCU_TLS (rcu_reader).ctr, tmp +RCU_GP_COUNT );
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 19 lines · * Lai Jiangshan ([email protected]) wrote:
Hi, Mathieu,
There is a big compatible problem in URCU which should be fix in next round.
LB: liburcu built on the system which has
sys_membarrier().
LU: liburcu built on the system which does NOT havesys_membarrier().LBM: liburcu-mb ....
LUM: liburcu-mb ...AB: application(-lliburcu) built on the system which has
sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT havesys_membarrier().ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinations
Quoted 7 lines
I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?
could you check it?
How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to
URCU_MB.
Hi Lai,
Good catch !
Would the following patch work for you ? It introduces RCU_MEMBARRIER_MB for implementation fallback, but keeps RCU_MEMBARRIER defined and used for the RCU flavor mapping file.
Thoughts ?
Thanks,
Mathieu
---
urcu.c +5 -5
@@ -62,14 +62,14 @@*/ #define RCU_QS_ACTIVE_ATTEMPTS 100 -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static int init_done;int rcu_has_sys_membarrier;void __attribute__((constructor))rcu_init (void );#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )void rcu_init (void ) { }
@@ -135,7 +135,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)urcu_die (ret); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier))
@@ -145,7 +145,7 @@ static void smp_mb_master(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {cmm_smp_mb ();
@@ -444,7 +444,7 @@ void rcu_unregister_thread(void)mutex_unlock (&rcu_gp_lock); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )void rcu_init (void ) {if (init_done)
urcu/map/urcu.h +0 -25
@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memb
urcu/static/urcu.h +7 -9
@@ -52,15 +52,14 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. Fallback to +* RCU_MEMBARRIER_MB otherwise. */ #if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB +#define RCU_MEMBARRIER_MB #endif -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )#include <syscall.h> /* If the headers do not support SYS_membarrier, statically use RCU_MB */
@@ -70,8 +69,7 @@ extern "C" {# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else -# undef RCU_MEMBARRIER -# define RCU_MB +# define RCU_MEMBARRIER_MB #endif #endif
@@ -178,7 +176,7 @@ static inline void rcu_debug_yield_init(void)#define MB_GROUP_ALL 0#define RCU_MB_GROUP MB_GROUP_ALL -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )extern int rcu_has_sys_membarrier;static inline void smp_mb_slave (int group)
@@ -190,7 +188,7 @@ static inline void smp_mb_slave(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static inline void smp_mb_slave (int group) {cmm_smp_mb ();
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20130508135343.GA8230@Krystal
Quoted 48 lines · Mathieu Desnoyers
Quoted 19 lines · * Lai Jiangshan ([email protected]) wrote:
Hi, Mathieu,
There is a big compatible problem in URCU which should be fix in next round.
LB: liburcu built on the system which has
sys_membarrier().
LU: liburcu built on the system which does NOT havesys_membarrier().LBM: liburcu-mb ....
LUM: liburcu-mb ...AB: application(-lliburcu) built on the system which has
sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT havesys_membarrier().ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinationsQuoted 7 lines
I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?
could you check it?
How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to
URCU_MB.Hi Lai,
Good catch !
Would the following patch work for you ? It introduces
RCU_MEMBARRIER_MBfor implementation fallback, but keepsRCU_MEMBARRIERdefined and used for the RCU flavor mapping file.Thoughts ?
Thanks,
Mathieu
---
urcu.c +0
-0
urcu.c line 67
Quoted 7 lines
urcu.c +1
-1@@ -62,14 +62,14 @@*/ #define RCU_QS_ACTIVE_ATTEMPTS 100 -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static int init_done;int rcu_has_sys_membarrier;
AB will use rcu_has_sys_membarrier, but LU don't have it.
RCU_MEMBARRIER_MB still introduces different ABI.
Quoted 134 lines
urcu.c +4
-4void __attribute__((constructor))rcu_init (void );#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )void rcu_init (void ) { }@@ -135,7 +135,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)urcu_die (ret); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier))@@ -145,7 +145,7 @@ static void smp_mb_master(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {cmm_smp_mb ();@@ -444,7 +444,7 @@ void rcu_unregister_thread(void)mutex_unlock (&rcu_gp_lock); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )void rcu_init (void ) {if (init_done)urcu/map/urcu.h +0
-25@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memburcu/static/urcu.h +7
-10@@ -52,15 +52,14 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. Fallback to +* RCU_MEMBARRIER_MB otherwise. */ #if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB +#define RCU_MEMBARRIER_MB #endif -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )#include <syscall.h> /* If the headers do not support SYS_membarrier, statically use RCU_MB */ @@ -70,8 +69,7 @@ extern "C" {# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else -# undef RCU_MEMBARRIER -# define RCU_MB +# define RCU_MEMBARRIER_MB #endif #endif @@ -178,7 +176,7 @@ static inline void rcu_debug_yield_init(void)#define MB_GROUP_ALL 0#define RCU_MB_GROUP MB_GROUP_ALL -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )extern int rcu_has_sys_membarrier;static inline void smp_mb_slave (int group)@@ -190,7 +188,7 @@ static inline void smp_mb_slave(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static inline void smp_mb_slave (int group) {cmm_smp_mb (); --Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
A message that is not in the archive (sent off-list or not yet received): CACvQF521cSR0_5K+aC79A+s9APypQOS8POEoP2cDRguguw6P9A@mail.gmail.com
* Lai Jiangshan ([email protected]) wrote:
[...]
Quoted 3 lines
AB will use
rcu_has_sys_membarrier, but LU don't have it.
RCU_MEMBARRIER_MBstill introduces different ABI.
Yes, you're right. How about the following then ?
---
urcu.c +8 -5
@@ -63,13 +63,16 @@#define RCU_QS_ACTIVE_ATTEMPTS 100#ifdef RCU_MEMBARRIER -static int init_done;int rcu_has_sys_membarrier; +#endif + +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB ) +static int init_done;void __attribute__((constructor))rcu_init (void );#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )void rcu_init (void ) { }
@@ -135,7 +138,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)urcu_die (ret); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier))
@@ -145,7 +148,7 @@ static void smp_mb_master(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {cmm_smp_mb ();
@@ -444,7 +447,7 @@ void rcu_unregister_thread(void)mutex_unlock (&rcu_gp_lock); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )void rcu_init (void ) {if (init_done)
urcu/map/urcu.h +0 -25
@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memb
urcu/static/urcu.h +5 -7
@@ -52,15 +52,14 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. Fallback to +* RCU_MEMBARRIER_MB otherwise. */ #if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB +#define RCU_MEMBARRIER_MB #endif -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )#include <syscall.h> /* If the headers do not support SYS_membarrier, statically use RCU_MB */
@@ -70,8 +69,7 @@ extern "C" {# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else -# undef RCU_MEMBARRIER -# define RCU_MB +# define RCU_MEMBARRIER_MB #endif #endif
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20130508181936.GA9919@Krystal
Quoted 121 lines · Mathieu Desnoyers
* Lai Jiangshan ([email protected]) wrote:
[...]Quoted 3 lines
AB will use
rcu_has_sys_membarrier, but LU don't have it.
RCU_MEMBARRIER_MBstill introduces different ABI.Yes, you're right. How about the following then ?
---
urcu.c +8
-5@@ -63,13 +63,16 @@#define RCU_QS_ACTIVE_ATTEMPTS 100#ifdef RCU_MEMBARRIER -static int init_done;int rcu_has_sys_membarrier; +#endif + +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB ) +static int init_done;void __attribute__((constructor))rcu_init (void );#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )void rcu_init (void ) { }@@ -135,7 +138,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)urcu_die (ret); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier))@@ -145,7 +148,7 @@ static void smp_mb_master(int group) }#endif -#ifdef RCU_MB +#if defined(RCU_MB ) || defined(RCU_MEMBARRIER_MB )static void smp_mb_master (int group) {cmm_smp_mb ();@@ -444,7 +447,7 @@ void rcu_unregister_thread(void)mutex_unlock (&rcu_gp_lock); } -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )void rcu_init (void ) {if (init_done)urcu/map/urcu.h +0
-25@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memburcu/static/urcu.h +4
-5@@ -52,15 +52,14 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. Fallback to +* RCU_MEMBARRIER_MB otherwise. */ #if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB +#define RCU_MEMBARRIER_MB #endif -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && !defined(RCU_MEMBARRIER_MB )#include <syscall.h> /* If the headers do not support SYS_membarrier, statically use RCU_MB */
urcu/static/urcu.h extern "C" { · line 75
Quoted 9 lines
urcu/static/urcu.h +1
-2@@ -70,8 +69,7 @@ extern "C" {# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else -# undef RCU_MEMBARRIER -# define RCU_MB +# define RCU_MEMBARRIER_MB #endif #endif
How about we also define membarrier(...) which just returns -ENOSYS when the system don't have SYS_membarrier?
A message that is not in the archive (sent off-list or not yet received): [email protected]
* Lai Jiangshan ([email protected]) wrote:
[...]
Quoted 2 lines
How about we also define membarrier(...) which just returns
-ENOSYSwhen the system don't haveSYS_membarrier?
Indeed, this vastly simplifies the patch:
Thoughts ?
---
urcu.c +1 -1
@@ -139,7 +139,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier)) -membarrier (MEMBARRIER_EXPEDITED ); + (void )membarrier (MEMBARRIER_EXPEDITED );else cmm_smp_mb (); }
urcu/map/urcu.h +0 -25
@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memb
urcu/static/urcu.h +10 -15
@@ -33,6 +33,7 @@#include <pthread.h> #include <unistd.h> #include <stdint.h> +#include <error.h> #include <urcu/compiler.h> #include <urcu/arch.h>
@@ -52,29 +53,23 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. */ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && defined(__linux__)#include <syscall.h> +#endif -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ +/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ #ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif +# define membarrier (...) -ENOSYS#endif +#define MEMBARRIER_EXPEDITED (1 << 0) +#define MEMBARRIER_DELAYED (1 << 1) +#define MEMBARRIER_QUERY (1 << 16) +/* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20130509122748.GA23531@Krystal
Quoted 97 lines · Mathieu Desnoyers
* Lai Jiangshan ([email protected]) wrote:
[...]Quoted 2 lines
How about we also define membarrier(...) which just returns
-ENOSYSwhen the system don't haveSYS_membarrier?Indeed, this vastly simplifies the patch:
Thoughts ?
---
urcu.c +1
-1@@ -139,7 +139,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier)) -membarrier (MEMBARRIER_EXPEDITED ); + (void )membarrier (MEMBARRIER_EXPEDITED );else cmm_smp_mb (); }urcu/map/urcu.h +0
-25@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memburcu/static/urcu.h +5
-12@@ -33,6 +33,7 @@#include <pthread.h> #include <unistd.h> #include <stdint.h> +#include <error.h> #include <urcu/compiler.h> #include <urcu/arch.h> @@ -52,29 +53,23 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. +* RCU_MEMBARRIER is only possibly available on Linux. */ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER +#if defined(RCU_MEMBARRIER ) && defined(__linux__)#include <syscall.h> +#endif -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ +/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ #ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16)# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__)#else
urcu/static/urcu.h extern "C" { · line 72
Quoted 10 lines
urcu/static/urcu.h +5
-3-# undef RCU_MEMBARRIER -# define RCU_MB -#endif +# define membarrier (...) -ENOSYS#endif +#define MEMBARRIER_EXPEDITED (1 << 0) +#define MEMBARRIER_DELAYED (1 << 1) +#define MEMBARRIER_QUERY (1 << 16) +
I think this code can be moved into urcu.c
Quoted 8 lines
urcu/static/urcu.h +0
-1/* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to -- Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): CACvQF53K3Qwj4JO3y82yhxGqrBfPdkgH2fXcj5SbYO0OTD+uSg@mail.gmail.com
* Lai Jiangshan ([email protected]) wrote:
[...]
Quoted 1 line
I think this code can be moved into
urcu.c
good point! How about this ?
---
urcu.c +19 -1
@@ -62,6 +62,24 @@*/ #define RCU_QS_ACTIVE_ATTEMPTS 100 +/* +* RCU_MEMBARRIER is only possibly available on Linux. +*/ +#if defined(RCU_MEMBARRIER ) && defined(__linux__) +#include <syscall.h> +#endif + +/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ +#ifdef SYS_membarrier +# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) +#else +# define membarrier (...) -ENOSYS +#endif + +#define MEMBARRIER_EXPEDITED (1 << 0) +#define MEMBARRIER_DELAYED (1 << 1) +#define MEMBARRIER_QUERY (1 << 16) +#ifdef RCU_MEMBARRIER static int init_done;int rcu_has_sys_membarrier;
@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier)) -membarrier (MEMBARRIER_EXPEDITED ); + (void )membarrier (MEMBARRIER_EXPEDITED );else cmm_smp_mb (); }
urcu/map/urcu.h +0 -25
@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memb
urcu/static/urcu.h +0 -24
@@ -52,30 +52,6 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - -/* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to * be only linked with the Userspace RCU library. This comes with a small
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com
A message that is not in the archive (sent off-list or not yet received): 20130509151018.GC27168@Krystal
Quoted 5 lines · Mathieu Desnoyers
* Lai Jiangshan ([email protected]) wrote:
[...]Quoted 1 line
I think this code can be moved into
urcu.cgood point! How about this ?
Hi, I don't see any obvious bad thing, but I can't carefully check it currently, It seems OK for me.
Could you make it some progress.
Thanks,
Lai
Quoted 112 lines
---
urcu.c +19
-1@@ -62,6 +62,24 @@*/ #define RCU_QS_ACTIVE_ATTEMPTS 100 +/* +* RCU_MEMBARRIER is only possibly available on Linux. +*/ +#if defined(RCU_MEMBARRIER ) && defined(__linux__) +#include <syscall.h> +#endif + +/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ +#ifdef SYS_membarrier +# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) +#else +# define membarrier (...) -ENOSYS +#endif + +#define MEMBARRIER_EXPEDITED (1 << 0) +#define MEMBARRIER_DELAYED (1 << 1) +#define MEMBARRIER_QUERY (1 << 16) +#ifdef RCU_MEMBARRIER static int init_done;int rcu_has_sys_membarrier;@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier)) -membarrier (MEMBARRIER_EXPEDITED ); + (void )membarrier (MEMBARRIER_EXPEDITED );else cmm_smp_mb (); }urcu/map/urcu.h +0
-25@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memburcu/static/urcu.h +0
-24@@ -52,30 +52,6 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - -/* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to * be only linked with the Userspace RCU library. This comes with a small
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 11 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 5 lines · Mathieu Desnoyers
* Lai Jiangshan ([email protected]) wrote:
[...]Quoted 1 line
I think this code can be moved into
urcu.cgood point! How about this ?
Hi, I don't see any obvious bad thing, but I can't carefully check it currently, It seems OK for me.
Could you make it some progress.
Thanks. Merged as:
commit 553b7eb9da3a223368abdf3f6a5ead35acf8a20b
Author: Mathieu Desnoyers <[email protected]> Date: Wed May 8 09:53:45 2013 -0400
Fix: membarrier fallback symbol conflict
Mathieu
Quoted 117 lines
Thanks,
LaiQuoted 112 lines
---
urcu.c +19
-1@@ -62,6 +62,24 @@*/ #define RCU_QS_ACTIVE_ATTEMPTS 100 +/* +* RCU_MEMBARRIER is only possibly available on Linux. +*/ +#if defined(RCU_MEMBARRIER ) && defined(__linux__) +#include <syscall.h> +#endif + +/* If the headers do not support SYS_membarrier, fall back on RCU_MB */ +#ifdef SYS_membarrier +# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) +#else +# define membarrier (...) -ENOSYS +#endif + +#define MEMBARRIER_EXPEDITED (1 << 0) +#define MEMBARRIER_DELAYED (1 << 1) +#define MEMBARRIER_QUERY (1 << 16) +#ifdef RCU_MEMBARRIER static int init_done;int rcu_has_sys_membarrier;@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)static void smp_mb_master (int group) {if (caa_likely (rcu_has_sys_membarrier)) -membarrier (MEMBARRIER_EXPEDITED ); + (void )membarrier (MEMBARRIER_EXPEDITED );else cmm_smp_mb (); }urcu/map/urcu.h +0
-25@@ -38,31 +38,6 @@#define RCU_MEMBARRIER #endif -/* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to -* RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif -#ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memburcu/static/urcu.h +0
-24@@ -52,30 +52,6 @@ extern "C" {#endif /* -* RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB -* otherwise. -*/ -#if !defined(__linux__) && defined(RCU_MEMBARRIER ) -#undef RCU_MEMBARRIER -#define RCU_MB -#endif - -#ifdef RCU_MEMBARRIER -#include <syscall.h> - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier (...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - -/* * This code section can only be included in LGPL 2.1 compatible source code. * See below for the function call wrappers which can be used in code meant to * be only linked with the Userspace RCU library. This comes with a small
Signature · 3 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com