Skip to contentSkip to conversation

urcu: avoid false sharing for rcu_gp_ctr

PATCH New Mathieu Desnoyers 4 participants 23 messages Dec 7, 2012 – May 16, 2013 21 messages not in the archive Kbuild

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw
Quoted 9 lines · * Lai Jiangshan ([email protected]) wrote:

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 195 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---

urcu.c +12 -21
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	smp_mb_master(RCU_MB_GROUP);
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_dec(&rcu_gp_futex);
+			uatomic_dec(&rcu_gp.futex);
 			/* Write futex before read reader_gp */
 			smp_mb_master(RCU_MB_GROUP);
 		}
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -357,10 +348,10 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity before
-	 * committing next rcu_gp_ctr update to memory. Failure to do so could
+	 * committing next rcu_gp.ctr update to memory. Failure to do so could
 	 * result in the writer waiting forever while new readers are always
 	 * accessing data (no progress).  Enforce compiler-order of load
-	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -372,13 +363,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for quiescent
+	 * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 	 * state. Failure to do so could result in the writer waiting forever
 	 * while new readers are always accessing data (no progress). Enforce
-	 * compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
+	 * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 	 */
 	cmm_barrier();
 
urcu/static/urcu.h +20 -14
index 973826a..0dd733e 100644
@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE	(1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK	(RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	/*
+	 * Global grace period counter.
+	 * Contains the current RCU_GP_CTR_PHASE.
+	 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
+	 * Written to only by writer with mutex taken.
+	 * Read by both writer and readers.
+	 */
+	unsigned long ctr;
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +239,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
 static inline void wake_up_gp(void)
 {
-	if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+	if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		smp_mb_slave(RCU_MB_GROUP);
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20121207163234.GB11975@Krystal

#3 lore raw

we can define rcu_gp_ctr and registry with aligned attribute, but it is not reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

Quoted 270 lines · Mathieu Desnoyers
Quoted 7 lines · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 8 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes

     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes

     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes

     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes

     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes

     4129809 nr_ops   3401767295
Quoted 9 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 10 lines
urcu.c +0 -8
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the reader

and the

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 12 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c void synchronize_rcu(void) · line 350

Quoted 12 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -357,10 +348,10 @@ void synchronize_rcu(void)
 
      /*
       * Must finish waiting for quiescent state for original parity

before

urcu.c void synchronize_rcu(void) · line 360

Quoted 1 line
urcu.c +0 -1
-      * committing next rcu_gp_ctr update to memory. Failure to do so

could

urcu.c void synchronize_rcu(void) · line 351

Quoted 1 line
urcu.c +1 -0
+      * committing next rcu_gp.ctr update to memory. Failure to do so

could

urcu.c void synchronize_rcu(void) · line 352

Quoted 1 line
urcu.c +0 -0
       * result in the writer waiting forever while new readers are

always

Quoted 7 lines
urcu.c +1 -1
       * accessing data (no progress).  Enforce compiler-order of load
-      * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+      * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
       */
      cmm_barrier();
 

urcu.c void synchronize_rcu(void) · line 378

Quoted 8 lines
urcu.c +1 -2
@@ -372,13 +363,13 @@ void synchronize_rcu(void)
      cmm_smp_mb();
 
      /* Switch parity: 0 -> 1, 1 -> 0 */
-     CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+     CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
      /*
-      * Must commit rcu_gp_ctr update to memory before waiting for

quiescent

urcu.c void synchronize_rcu(void) · line 369

Quoted 1 line
urcu.c +1 -0
+      * Must commit rcu_gp.ctr update to memory before waiting for

quiescent

urcu.c void synchronize_rcu(void) · line 370

Quoted 1 line
urcu.c +0 -0
       * state. Failure to do so could result in the writer waiting

forever

urcu.c void synchronize_rcu(void) · line 371

Quoted 1 line
urcu.c +0 -0
       * while new readers are always accessing data (no progress).

Enforce

urcu.c void synchronize_rcu(void) · line 381

Quoted 1 line
urcu.c +0 -1
-      * compiler-order of store to rcu_gp_ctr before load rcu_reader

ctr.

urcu.c void synchronize_rcu(void) · line 372

Quoted 1 line
urcu.c +1 -0
+      * compiler-order of store to rcu_gp.ctr before load rcu_reader

ctr.

Quoted 8 lines
urcu.c +0 -0
       */
      cmm_barrier();
 
urcu/static/urcu.h +0 -0
index 973826a..0dd733e 100644

urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 218

Quoted 6 lines
urcu/static/urcu.h +0 -3
@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE     (1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register

dependencies

urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 220

Quoted 8 lines
urcu/static/urcu.h +5 -3
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+     /*
+      * Global grace period counter.
+      * Contains the current RCU_GP_CTR_PHASE.
+      * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast

path.

Quoted 13 lines
urcu/static/urcu.h +9 -0
+      * Written to only by writer with mutex taken.
+      * Read by both writer and readers.
+      */
+     unsigned long ctr;
+
+     int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
      /* Data used by both reader and synchronize_rcu() */

urcu/static/urcu.h struct rcu_reader { · line 243

Quoted 7 lines
urcu/static/urcu.h +0 -2
@@ -231,16 +239,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent

threads.

urcu/static/urcu.h static inline enum rcu_state

Quoted 13 lines
urcu/static/urcu.h +3 -3
  */
 static inline void wake_up_gp(void)
 {
-     if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-             uatomic_set(&rcu_gp_futex, 0);
-             futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+     if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+             uatomic_set(&rcu_gp.futex, 0);
+             futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
                    NULL, NULL, 0);
      }
 }
@@ -256,13 +262,13 @@ static inline enum rcu_state

rcu_reader_state(unsigned long *ctr)

urcu/static/urcu.h static inline enum rcu_state · line 272

Quoted 13 lines
urcu/static/urcu.h +2 -2
      v = CMM_LOAD_SHARED(*ctr);
      if (!(v & RCU_GP_CTR_NEST_MASK))
              return RCU_READER_INACTIVE;
-     if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+     if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
              return RCU_READER_ACTIVE_CURRENT;
      return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count

of

urcu/static/urcu.h static inline enum rcu_state · line 273

Quoted 1 line
urcu/static/urcu.h +0 -0
  * _rcu_read_lock() nesting, and a lower-order bit that contains either

zero

urcu/static/urcu.h static inline enum rcu_state

Quoted 2 lines
urcu/static/urcu.h +0 -0
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +277,7 @@ static inline enum rcu_state

rcu_reader_state(unsigned long *ctr)

urcu/static/urcu.h static inline enum rcu_state · line 274

Quoted 4 lines
urcu/static/urcu.h +0 -1
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
      if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-             _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,

_CMM_LOAD_SHARED(rcu_gp_ctr));

urcu/static/urcu.h static inline enum rcu_state · line 280

Quoted 1 line
urcu/static/urcu.h +1 -0
+             _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,

_CMM_LOAD_SHARED(rcu_gp.ctr));

urcu/static/urcu.h static inline enum rcu_state · line 283

Quoted 3 lines
urcu/static/urcu.h +0 -0
              smp_mb_slave(RCU_MB_GROUP);
      } else
              _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp +

RCU_GP_COUNT);

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Quoted 5 lines

_______________________________________________
lttng-dev mailing list
[email protected] <javascript:;>
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): CACvQF50Y2wC3t=VMJ==574aHVvod+ipGA1K5hR97r_dRbcegkw@mail.gmail.com

#5 lore raw
Quoted 6 lines · * Lai Jiangshan ([email protected]) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is not reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Thanks,

Mathieu

Quoted 278 lines · Mathieu Desnoyers
Quoted 276 lines · * Lai Jiangshan ([email protected] <javascript:;>) wrote:
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too (keeping qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if we choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 8 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes

     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes

     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes

     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes

     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes

     4129809 nr_ops   3401767295
Quoted 9 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 10 lines
urcu.c +0 -8
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the reader

and the

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 12 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c void synchronize_rcu(void) · line 350

Quoted 12 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -357,10 +348,10 @@ void synchronize_rcu(void)
 
      /*
       * Must finish waiting for quiescent state for original parity

before

urcu.c void synchronize_rcu(void) · line 360

Quoted 1 line
urcu.c +0 -1
-      * committing next rcu_gp_ctr update to memory. Failure to do so

could

urcu.c void synchronize_rcu(void) · line 351

Quoted 1 line
urcu.c +1 -0
+      * committing next rcu_gp.ctr update to memory. Failure to do so

could

urcu.c void synchronize_rcu(void) · line 352

Quoted 1 line
urcu.c +0 -0
       * result in the writer waiting forever while new readers are

always

Quoted 7 lines
urcu.c +1 -1
       * accessing data (no progress).  Enforce compiler-order of load
-      * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+      * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
       */
      cmm_barrier();
 

urcu.c void synchronize_rcu(void) · line 378

Quoted 8 lines
urcu.c +1 -2
@@ -372,13 +363,13 @@ void synchronize_rcu(void)
      cmm_smp_mb();
 
      /* Switch parity: 0 -> 1, 1 -> 0 */
-     CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+     CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
      /*
-      * Must commit rcu_gp_ctr update to memory before waiting for

quiescent

urcu.c void synchronize_rcu(void) · line 369

Quoted 1 line
urcu.c +1 -0
+      * Must commit rcu_gp.ctr update to memory before waiting for

quiescent

urcu.c void synchronize_rcu(void) · line 370

Quoted 1 line
urcu.c +0 -0
       * state. Failure to do so could result in the writer waiting

forever

urcu.c void synchronize_rcu(void) · line 371

Quoted 1 line
urcu.c +0 -0
       * while new readers are always accessing data (no progress).

Enforce

urcu.c void synchronize_rcu(void) · line 381

Quoted 1 line
urcu.c +0 -1
-      * compiler-order of store to rcu_gp_ctr before load rcu_reader

ctr.

urcu.c void synchronize_rcu(void) · line 372

Quoted 1 line
urcu.c +1 -0
+      * compiler-order of store to rcu_gp.ctr before load rcu_reader

ctr.

Quoted 8 lines
urcu.c +0 -0
       */
      cmm_barrier();
 
urcu/static/urcu.h +0 -0
index 973826a..0dd733e 100644

urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 218

Quoted 6 lines
urcu/static/urcu.h +0 -3
@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE     (1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register

dependencies

urcu/static/urcu.h static inline void smp_mb_slave(int group) · line 220

Quoted 8 lines
urcu/static/urcu.h +5 -3
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+     /*
+      * Global grace period counter.
+      * Contains the current RCU_GP_CTR_PHASE.
+      * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast

path.

Quoted 13 lines
urcu/static/urcu.h +9 -0
+      * Written to only by writer with mutex taken.
+      * Read by both writer and readers.
+      */
+     unsigned long ctr;
+
+     int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
      /* Data used by both reader and synchronize_rcu() */

urcu/static/urcu.h struct rcu_reader { · line 243

Quoted 7 lines
urcu/static/urcu.h +0 -2
@@ -231,16 +239,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent

threads.

urcu/static/urcu.h static inline enum rcu_state

Quoted 13 lines
urcu/static/urcu.h +3 -3
  */
 static inline void wake_up_gp(void)
 {
-     if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-             uatomic_set(&rcu_gp_futex, 0);
-             futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+     if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+             uatomic_set(&rcu_gp.futex, 0);
+             futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
                    NULL, NULL, 0);
      }
 }
@@ -256,13 +262,13 @@ static inline enum rcu_state

rcu_reader_state(unsigned long *ctr)

urcu/static/urcu.h static inline enum rcu_state · line 272

Quoted 13 lines
urcu/static/urcu.h +2 -2
      v = CMM_LOAD_SHARED(*ctr);
      if (!(v & RCU_GP_CTR_NEST_MASK))
              return RCU_READER_INACTIVE;
-     if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+     if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
              return RCU_READER_ACTIVE_CURRENT;
      return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count

of

urcu/static/urcu.h static inline enum rcu_state · line 273

Quoted 1 line
urcu/static/urcu.h +0 -0
  * _rcu_read_lock() nesting, and a lower-order bit that contains either

zero

urcu/static/urcu.h static inline enum rcu_state

Quoted 2 lines
urcu/static/urcu.h +0 -0
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +277,7 @@ static inline enum rcu_state

rcu_reader_state(unsigned long *ctr)

urcu/static/urcu.h static inline enum rcu_state · line 274

Quoted 4 lines
urcu/static/urcu.h +0 -1
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
      if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-             _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,

_CMM_LOAD_SHARED(rcu_gp_ctr));

urcu/static/urcu.h static inline enum rcu_state · line 280

Quoted 1 line
urcu/static/urcu.h +1 -0
+             _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr,

_CMM_LOAD_SHARED(rcu_gp.ctr));

urcu/static/urcu.h static inline enum rcu_state · line 283

Quoted 3 lines
urcu/static/urcu.h +0 -0
              smp_mb_slave(RCU_MB_GROUP);
      } else
              _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp +

RCU_GP_COUNT);

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected] <javascript:;>
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20121207165748.GA12338@Krystal

#7 lore raw
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

Quoted 151 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 30 lines
Quoted 30 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): CACvQF50V_9T6T3j4AwbdBuEiHHbcxdNGgP1WF5T4OHpXZyMgEA@mail.gmail.com

#9 lore raw
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Quoted 6 lines

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:

typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));

I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.

I guess this is a case where we could bump the library version number and do things properly.

Let's think a bit more about it, anyone else has comments on this ?

Thanks,

Mathieu

Quoted 162 lines
Quoted 162 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 27 lines
Quoted 27 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 5 lines
Quoted 5 lines

urcu.c static void wait_for_readers(struct cds_list_head · line 282

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20121207172252.GA12525@Krystal

#11 lore raw
Quoted 30 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Quoted 6 lines

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:

typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));

Quoted 8 lines

I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.

I guess this is a case where we could bump the library version number and do things properly.

Let's think a bit more about it, anyone else has comments on this ?

If you really want the alignment and padding, they should go into a structure. ;-)

Thanx, Paul

Quoted 173 lines

Thanks,

Mathieu

Quoted 162 lines
Quoted 162 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 27 lines
Quoted 27 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 5 lines
Quoted 5 lines

urcu.c static void wait_for_readers(struct cds_list_head · line 282

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): [email protected]

#13 lore raw
Quoted 35 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 34 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Quoted 6 lines

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:

typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));

I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.

Quoted 7 lines
Quoted 4 lines

I guess this is a case where we could bump the library version number and do things properly.

Let's think a bit more about it, anyone else has comments on this ?

If you really want the alignment and padding, they should go into a structure. ;-)

OK, so we would bump to liburcu2 then for the 0.8 release ?

Thanks,

Mathieu

Quoted 177 lines

Thanx, Paul

Quoted 173 lines

Thanks,

Mathieu

Quoted 162 lines
Quoted 162 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 27 lines
Quoted 27 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 5 lines
Quoted 5 lines

urcu.c static void wait_for_readers(struct cds_list_head · line 282

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20121210180111.GA11209@Krystal

#15 lore raw
Quoted 39 lines · Mathieu Desnoyers
Quoted 38 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 37 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Quoted 6 lines

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:

typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));

I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.

I guess this is a case where we could bump the library version number and do things properly.

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

Let's think a bit more about it, anyone else has comments on this ?

If you really want the alignment and padding, they should go into a structure. ;-)

OK, so we would bump to liburcu2 then for the 0.8 release ?

I don't believe that this optimization by itself warrants a version bump. Perhaps hold it in reserve when the next need for a version bump comes along.

So, how to do this without a version bump? One approach would be to use linker-script magic (yeah, now -that- will be portable!). Or if the alignment attributes happen to do the right thing on popular gcc versions, go with them until the next version bump allows us to do it more deterministically.

Thanx, Paul

Quoted 188 lines

Thanks,

Mathieu

Quoted 177 lines

Thanx, Paul

Quoted 173 lines

Thanks,

Mathieu

Quoted 162 lines
Quoted 162 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 27 lines
Quoted 27 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 5 lines
Quoted 5 lines

urcu.c static void wait_for_readers(struct cds_list_head · line 282

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): [email protected]

#17 lore raw
Quoted 45 lines · Paul E. McKenney
Quoted 44 lines · Mathieu Desnoyers
Quoted 42 lines · * Paul E. McKenney ([email protected]) wrote:
Quoted 30 lines · Mathieu Desnoyers
Quoted 15 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 10 lines · Mathieu Desnoyers
Quoted 1 line · * Lai Jiangshan ([email protected] <javascript:;>) wrote:

we can define rcu_gp_ctr and registry with aligned attribute, but it is

not

Quoted 5 lines

reliable way

We need only this:
unsigned long rcu_gp_ctr __attribute((aligned and padded(don't put other var next to it except the futex)))

In which situation would this be unreliable ?

Quoted 6 lines

int a;
int b __attribute__((aligned));
int c;

b and c will be in the same line, even we define c as aligned too, the compiler/linker may put a next to b, thus a and b in the same line

So if our goal is to have rcu_gp_ctr and rcu_gp_futex on the same cache line, which is different from that of the registry, we could do:

typeA rcu_gp_ctr __attribute__((aligned(...)));
typeB rcu_gp_futex;
typeC registry __attribute__((aligned(...)));

Quoted 8 lines

I would expect the compiler won't typically reorder rcu_gp_futex and registry. But I guess there is no guarantee it is going to be always true given by the C99 standard.

I guess this is a case where we could bump the library version number and do things properly.

Let's think a bit more about it, anyone else has comments on this ?

If you really want the alignment and padding, they should go into a structure. ;-)

Quoted 8 lines
Quoted 1 line

OK, so we would bump to liburcu2 then for the 0.8 release ?

I don't believe that this optimization by itself warrants a version bump. Perhaps hold it in reserve when the next need for a version bump comes along.

So, how to do this without a version bump? One approach would be to use linker-script magic

I already did it, send soon.

Quoted 197 lines

(yeah, now -that- will be portable!). Or if
the alignment attributes happen to do the right thing on popular gcc versions, go with them until the next version bump allows us to do it more deterministically.

Thanx, Paul

Quoted 188 lines

Thanks,

Mathieu

Quoted 177 lines

Thanx, Paul

Quoted 173 lines

Thanks,

Mathieu

Quoted 162 lines
Quoted 162 lines

Thanks,

Mathieu

Quoted 4 lines · Mathieu Desnoyers
Quoted 2 lines

* Lai Jiangshan ([email protected] <javascript:;> <javascript:;>)

Quoted 9 lines · wrote:
Quoted 9 lines
Quoted 7 lines

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but they always be accessed at almost the same time, so we also move

rcu_gp_futex

Quoted 1 line

to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or

Quoted 8 lines
Quoted 8 lines

cache-missing

Quoted 1 line

of read site.

Hi Lai,

I agree on the goal: placing registry and rcu_gp_ctr on different cache-lines. And yes, it makes sense to put rcu_gp_ctr and rcu_gp_futex on the same cache-line. I agree that the next patch is fine too

(keeping

Quoted 7 lines
Quoted 7 lines

qsbr and other urcu similar). This is indeed what I try to ensure myself.

I'm just concerned that this patch seems to break ABI compability for
liburcu: the read-side, within applications, would have to be recompiled. So I guess we should decide if we do this change in a way that does not break the ABI (e.g. not introducing a structure), or if

we

Quoted 10 lines
Quoted 10 lines

choose to bump the library version number.

Thoughts ?

Thanks,

Mathieu

Quoted 2 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330

nr_writes

Quoted 4 lines
Quoted 4 lines
     3390219 nr_ops   2103675549
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562

nr_writes

Quoted 4 lines
Quoted 4 lines
     3529478 nr_ops   1623398040
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038

nr_writes

Quoted 8 lines
Quoted 8 lines
     3469334 nr_ops   1952536372
Quoted 6 lines

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848

nr_writes

Quoted 4 lines
Quoted 4 lines
     4903248 nr_ops   3385095096
Quoted 2 lines

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur

0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486

nr_writes

Quoted 11 lines
Quoted 11 lines
     4129809 nr_ops   3401767295
Quoted 10 lines
Singed-by Lai Jiangshan <[email protected] <javascript:;><javascript:;>>

---

urcu.c +0 -0
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
Quoted 9 lines
Quoted 9 lines

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 93

Quoted 9 lines
urcu.c +0 -8
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer

and

Quoted 7 lines
Quoted 7 lines

readers.

urcu.c void __attribute__((destructor)) rcu_exit(void); · line 89

Quoted 6 lines
urcu.c +1 -2
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the

reader

Quoted 12 lines
Quoted 12 lines

and the

Quoted 11 lines
urcu.c +2 -2
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
      /* Read reader_gp before read futex */
      smp_mb_master(RCU_MB_GROUP);
-     if (uatomic_read(&rcu_gp_futex) == -1)
-             futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+     if (uatomic_read(&rcu_gp.futex) == -1)
+             futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
                    NULL, NULL, 0);
 }
 
Quoted 1 line
Quoted 1 line

urcu.c static void wait_for_readers(struct

Quoted 1 line
urcu.c +0 -0
@@ -232,12 +223,12 @@ static void wait_for_readers(struct

cds_list_head

Quoted 27 lines
Quoted 27 lines

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 15 lines
urcu.c +2 -2
      /*
       * Wait for each thread URCU_TLS(rcu_reader).ctr to either
       * indicate quiescence (not nested), or observe the current
-      * rcu_gp_ctr value.
+      * rcu_gp.ctr value.
       */
      for (;;) {
              wait_loops++;
              if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-                     uatomic_dec(&rcu_gp_futex);
+                     uatomic_dec(&rcu_gp.futex);
                      /* Write futex before read reader_gp */
                      smp_mb_master(RCU_MB_GROUP);
              }
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

urcu.c static void wait_for_readers(struct cds_list_head

Quoted 9 lines
urcu.c +1 -1
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                              smp_mb_master(RCU_MB_GROUP);
-                             uatomic_set(&rcu_gp_futex, 0);
+                             uatomic_set(&rcu_gp.futex, 0);
                      }
                      break;
              } else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head

*input_readers,

Quoted 5 lines
Quoted 5 lines

urcu.c static void wait_for_readers(struct cds_list_head · line 282

Quoted 3 lines
urcu.c +0 -0
                      if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
                              /* Read reader_gp before write futex */
                            > > [email protected]<javascript:;><javascript:;>

http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com
#18 lore raw

PATCH v2 1/1 urcu: avoid false sharing for rcu_gp_ctr

In reply to a message not in the archive

From 66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90 Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing for rcu_gp_ctr

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---
 Makefile.am              |   10 +++++-----
 compat_rcu_gp.lds        |   25 +++++++++++++++++++++++++
 compat_rcu_gp_bp.lds     |   24 ++++++++++++++++++++++++
 compat_rcu_gp_mb.lds     |   25 +++++++++++++++++++++++++
 compat_rcu_gp_qsbr.lds   |   25 +++++++++++++++++++++++++
 compat_rcu_gp_signal.lds |   25 +++++++++++++++++++++++++
 urcu-bp.c                |   14 ++++----------
 urcu-qsbr.c              |   35 +++++++++++++++--------------------
 urcu.c                   |   34 +++++++++++++---------------------
 urcu/map/urcu-bp.h       |    3 +--
 urcu/map/urcu-qsbr.h     |    3 +--
 urcu/map/urcu.h          |    9 +++------
 urcu/static/urcu-bp.h    |   23 ++++++++++++++---------
 urcu/static/urcu-qsbr.h  |   39 ++++++++++++++++++++++++---------------
 urcu/static/urcu.h       |   38 ++++++++++++++++++++++++--------------
 15 files changed, 228 insertions(+), 104 deletions(-)
 create mode 100644 compat_rcu_gp.lds
 create mode 100644 compat_rcu_gp_bp.lds
 create mode 100644 compat_rcu_gp_mb.lds
 create mode 100644 compat_rcu_gp_qsbr.lds
 create mode 100644 compat_rcu_gp_signal.lds
Makefile.am +5 -5
index 0a4d357..b2b7b6a 100644
@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \
 #
 liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
 
-liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT)
 liburcu_la_LIBADD = liburcu-common.la
 
-liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT)
+liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT)
 liburcu_qsbr_la_LIBADD = liburcu-common.la
 
-liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT)
 liburcu_mb_la_CFLAGS = -DRCU_MB
 liburcu_mb_la_LIBADD = liburcu-common.la
 
-liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_signal.lds $(COMPAT)
 liburcu_signal_la_CFLAGS = -DRCU_SIGNAL
 liburcu_signal_la_LIBADD = liburcu-common.la
 
-liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT)
+liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c compat_rcu_gp_bp.lds $(COMPAT)
 liburcu_bp_la_LIBADD = liburcu-common.la
 
 liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \
compat_rcu_gp.lds new file +25 -0
new file mode 100644
index 0000000..506a4ae
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp.lds
+ *
+ * Linker Script for compatibility of liburcu
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_memb = rcu_gp_memb;
+rcu_gp_futex_memb = rcu_gp_memb + 8;
+
compat_rcu_gp_bp.lds new file +24 -0
new file mode 100644
index 0000000..6ab7dfa
@@ -0,0 +1,24 @@
+/*
+ * compat_rcu_gp_bp.lds
+ *
+ * Linker Script for compatibility of liburcu-bp
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_bp = rcu_gp_bp;
+
compat_rcu_gp_mb.lds new file +25 -0
new file mode 100644
index 0000000..8313588
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_mb.lds
+ *
+ * Linker Script for compatibility of liburcu-mb
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_mb = rcu_gp_mb;
+rcu_gp_futex_mb = rcu_gp_mb + 8;
+
compat_rcu_gp_qsbr.lds new file +25 -0
new file mode 100644
index 0000000..d27e163
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_qsbr.lds
+ *
+ * Linker Script for compatibility of liburcu-qsbr
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_qsbr = rcu_gp_qsbr;
+rcu_gp_futex_qsbr = rcu_gp_qsbr + 8;
+
compat_rcu_gp_signal.lds new file +25 -0
new file mode 100644
index 0000000..f0a447d
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_signal.lds
+ *
+ * Linker Script for compatibility of liburcu-signal
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_signal = rcu_gp_signal;
+rcu_gp_futex_signal = rcu_gp_signal + 8;
+
urcu-bp.c +4 -10
index f99c0e5..2a2455b 100644
@@ -103,6 +103,8 @@ void *mremap_wrapper(void *old_address, size_t old_size,
 void __attribute__((destructor)) rcu_bp_exit(void);
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
+/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 #ifdef DEBUG_YIELD
 unsigned int rcu_yield_active;
@@ -110,14 +112,6 @@ DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);
 #endif
 
 /*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
-
-/*
  * Pointer to registry elements. Written to only by each individual reader. Read
  * by both the reader and the writers.
  */
@@ -174,7 +168,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
@@ -250,7 +244,7 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
 	 * Must commit qparity update to memory before waiting for other parity
urcu-qsbr.c +15 -20
index 3c2c65d..5e2510d 100644
@@ -53,13 +53,8 @@
 void __attribute__((destructor)) rcu_exit(void);
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- */
-unsigned long rcu_gp_ctr = RCU_GP_ONLINE;
+/* Init the ctr as RCU_GP_ONLINE, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { { .ctr = RCU_GP_ONLINE } };
 
 /*
  * Active attempts to check for reader Q.S. before calling futex().
@@ -118,8 +113,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	cmm_smp_rmb();
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_noasync(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_noasync(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -133,12 +128,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (offline), or for them to observe the
-	 * current rcu_gp_ctr value.
+	 * current rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_set(&rcu_gp_futex, -1);
+			uatomic_set(&rcu_gp.futex, -1);
 			/*
 			 * Write futex before write waiting (the other side
 			 * reads them in the opposite order).
@@ -177,7 +172,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				cmm_smp_mb();
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -253,11 +248,11 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity
-	 * before committing next rcu_gp_ctr update to memory. Failure
+	 * before committing next rcu_gp.ctr update to memory. Failure
 	 * to do so could result in the writer waiting forever while new
 	 * readers are always accessing data (no progress).  Enforce
 	 * compiler-order of load URCU_TLS(rcu_reader).ctr before store
-	 * to rcu_gp_ctr.
+	 * to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -269,13 +264,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for
+	 * Must commit rcu_gp.ctr update to memory before waiting for
 	 * quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data
-	 * (no progress). Enforce compiler-order of store to rcu_gp_ctr
+	 * (no progress). Enforce compiler-order of store to rcu_gp.ctr
 	 * before load URCU_TLS(rcu_reader).ctr.
 	 */
 	cmm_barrier();
@@ -353,13 +348,13 @@ void synchronize_rcu(void)
 		goto out;
 
 	/* Increment current G.P. */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr + RCU_GP_CTR);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for
+	 * Must commit rcu_gp.ctr update to memory before waiting for
 	 * quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data
-	 * (no progress). Enforce compiler-order of store to rcu_gp_ctr
+	 * (no progress). Enforce compiler-order of store to rcu_gp.ctr
 	 * before load URCU_TLS(rcu_reader).ctr.
 	 */
 	cmm_barrier();
urcu.c +13 -21
index 15def09..782c42e 100644
@@ -83,16 +83,8 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { { .ctr = RCU_GP_COUNT } };
 
 /*
  * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +209,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	smp_mb_master(RCU_MB_GROUP);
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -232,12 +224,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_dec(&rcu_gp_futex);
+			uatomic_dec(&rcu_gp.futex);
 			/* Write futex before read reader_gp */
 			smp_mb_master(RCU_MB_GROUP);
 		}
@@ -270,7 +262,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -289,7 +281,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -357,10 +349,10 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity before
-	 * committing next rcu_gp_ctr update to memory. Failure to do so could
+	 * committing next rcu_gp.ctr update to memory. Failure to do so could
 	 * result in the writer waiting forever while new readers are always
 	 * accessing data (no progress).  Enforce compiler-order of load
-	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -372,13 +364,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for quiescent
+	 * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 	 * state. Failure to do so could result in the writer waiting forever
 	 * while new readers are always accessing data (no progress). Enforce
-	 * compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
+	 * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 	 */
 	cmm_barrier();
 
urcu/map/urcu-bp.h +1 -2
index 63582fd..0464d3b 100644
@@ -44,8 +44,7 @@
 #define rcu_exit			rcu_exit_bp
 #define synchronize_rcu			synchronize_rcu_bp
 #define rcu_reader			rcu_reader_bp
-#define rcu_gp_ctr			rcu_gp_ctr_bp
-#define rcu_gp_futex			rcu_gp_futex_bp	/* unused */
+#define rcu_gp				rcu_gp_bp
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_bp
 #define get_call_rcu_thread		get_call_rcu_thread_bp
urcu/map/urcu-qsbr.h +1 -2
index f0aecc9..2c22baf 100644
@@ -47,8 +47,7 @@
 #define rcu_exit			rcu_exit_qsbr
 #define synchronize_rcu			synchronize_rcu_qsbr
 #define rcu_reader			rcu_reader_qsbr
-#define rcu_gp_ctr			rcu_gp_ctr_qsbr
-#define rcu_gp_futex			rcu_gp_futex_qsbr
+#define rcu_gp				rcu_gp_qsbr
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_qsbr
 #define get_call_rcu_thread		get_call_rcu_thread_qsbr
urcu/map/urcu.h +3 -6
index 759cb41..5ef3d4f 100644
@@ -75,8 +75,7 @@
 #define rcu_exit			rcu_exit_memb
 #define synchronize_rcu			synchronize_rcu_memb
 #define rcu_reader			rcu_reader_memb
-#define rcu_gp_ctr			rcu_gp_ctr_memb
-#define rcu_gp_futex			rcu_gp_futex_memb
+#define rcu_gp				rcu_gp_memb
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_memb
 #define get_call_rcu_thread		get_call_rcu_thread_memb
@@ -121,8 +120,7 @@
 #define rcu_exit			rcu_exit_sig
 #define synchronize_rcu			synchronize_rcu_sig
 #define rcu_reader			rcu_reader_sig
-#define rcu_gp_ctr			rcu_gp_ctr_sig
-#define rcu_gp_futex			rcu_gp_futex_sig
+#define rcu_gp				rcu_gp_sig
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_sig
 #define get_call_rcu_thread		get_call_rcu_thread_sig
@@ -164,8 +162,7 @@
 #define rcu_exit			rcu_exit_mb
 #define synchronize_rcu			synchronize_rcu_mb
 #define rcu_reader			rcu_reader_mb
-#define rcu_gp_ctr			rcu_gp_ctr_mb
-#define rcu_gp_futex			rcu_gp_futex_mb
+#define rcu_gp				rcu_gp_mb
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_mb
 #define get_call_rcu_thread		get_call_rcu_thread_mb
urcu/static/urcu-bp.h +14 -9
index c7f5326..a773a01 100644
@@ -130,12 +130,17 @@ static inline void rcu_debug_yield_init(void)
  */
 extern void rcu_bp_register(void);
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	/*
+	 * Global quiescent period counter with low-order bits unused.
+	 * Contains the current RCU_GP_CTR_PHASE.
+	 * Written to only by writer with mutex taken.
+	 * Read by both writer and readers.
+	 */
+	unsigned long ctr;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -166,13 +171,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -181,7 +186,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		cmm_smp_mb();
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp + RCU_GP_COUNT);
urcu/static/urcu-qsbr.h +24 -15
index f6e5580..d7a1e1c 100644
@@ -119,12 +119,23 @@ static inline void rcu_debug_yield_init(void)
 #define RCU_GP_ONLINE		(1UL << 0)
 #define RCU_GP_CTR		(1UL << 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	union {
+		/*
+		 * Global quiescent period counter with low-order bits unused.
+		 * Using a int rather than a char to eliminate false register
+		 * dependencies causing stalls on some architectures.
+		 */
+		unsigned long ctr;
+
+		/* ensure the futex is located at offset=8 */
+		int64_t __unused;
+	};
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -137,8 +148,6 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
@@ -147,10 +156,10 @@ static inline void wake_up_gp(void)
 	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
 		cmm_smp_mb();
-		if (uatomic_read(&rcu_gp_futex) != -1)
+		if (uatomic_read(&rcu_gp.futex) != -1)
 			return;
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_noasync(&rcu_gp_futex, FUTEX_WAKE, 1,
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_noasync(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -162,7 +171,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!v)
 		return RCU_READER_INACTIVE;
-	if (v == rcu_gp_ctr)
+	if (v == rcu_gp.ctr)
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
@@ -216,7 +225,7 @@ static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
  * to be invoked directly from non-LGPL code.
  *
  * We skip the memory barriers and gp store if our local ctr already
- * matches the global rcu_gp_ctr value: this is OK because a prior
+ * matches the global rcu_gp.ctr value: this is OK because a prior
  * _rcu_quiescent_state() or _rcu_thread_online() already updated it
  * within our thread, so we have no quiescent state to report.
  */
@@ -224,7 +233,7 @@ static inline void _rcu_quiescent_state(void)
 {
 	unsigned long gp_ctr;
 
-	if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp_ctr)) == URCU_TLS(rcu_reader).ctr)
+	if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp.ctr)) == URCU_TLS(rcu_reader).ctr)
 		return;
 	_rcu_quiescent_state_update_and_wakeup(gp_ctr);
 }
@@ -257,7 +266,7 @@ static inline void _rcu_thread_offline(void)
 static inline void _rcu_thread_online(void)
 {
 	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
-	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
+	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp.ctr));
 	cmm_smp_mb();
 }
 
urcu/static/urcu.h +24 -14
index 973826a..d23e673 100644
@@ -213,12 +213,24 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE	(1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK	(RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	union {
+		/*
+		 * Global grace period counter.
+		 * Contains the current RCU_GP_CTR_PHASE.
+		 * Written to only by writer with mutex taken.
+		 * Read by both writer and readers.
+		 */
+		unsigned long ctr;
+
+		/* ensure the futex is located at offset=8 */
+		int64_t __unused;
+	};
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +243,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
 static inline void wake_up_gp(void)
 {
-	if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+	if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -256,13 +266,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +281,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		smp_mb_slave(RCU_MB_GROUP);
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
Signature · 1 line
1.7.4.4

A message that is not in the archive (sent off-list or not yet received): [email protected]

#20 lore raw
Quoted 62 lines · * Lai Jiangshan ([email protected]) wrote:

From 66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90 Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing for rcu_gp_ctr

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---
 Makefile.am              |   10 +++++-----
 compat_rcu_gp.lds        |   25 +++++++++++++++++++++++++
 compat_rcu_gp_bp.lds     |   24 ++++++++++++++++++++++++
 compat_rcu_gp_mb.lds     |   25 +++++++++++++++++++++++++
 compat_rcu_gp_qsbr.lds   |   25 +++++++++++++++++++++++++
 compat_rcu_gp_signal.lds |   25 +++++++++++++++++++++++++
 urcu-bp.c                |   14 ++++----------
 urcu-qsbr.c              |   35 +++++++++++++++--------------------
 urcu.c                   |   34 +++++++++++++---------------------
 urcu/map/urcu-bp.h       |    3 +--
 urcu/map/urcu-qsbr.h     |    3 +--
 urcu/map/urcu.h          |    9 +++------
 urcu/static/urcu-bp.h    |   23 ++++++++++++++---------
 urcu/static/urcu-qsbr.h  |   39 ++++++++++++++++++++++++---------------
 urcu/static/urcu.h       |   38 ++++++++++++++++++++++++--------------
 15 files changed, 228 insertions(+), 104 deletions(-)
 create mode 100644 compat_rcu_gp.lds
 create mode 100644 compat_rcu_gp_bp.lds
 create mode 100644 compat_rcu_gp_mb.lds
 create mode 100644 compat_rcu_gp_qsbr.lds
 create mode 100644 compat_rcu_gp_signal.lds
Makefile.am +0 -0
index 0a4d357..b2b7b6a 100644

Makefile.am lib_LTLIBRARIES = liburcu-common.la \ · line 59

Quoted 5 lines
Makefile.am +1 -1
@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \
 #
 liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
 
-liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT)

Hrm, I tried your patch with "make V=1", and have not found any use of those "lds" files. I'm not sure adding them as SOURCES is enough.

Quoted 550 lines
Makefile.am +4 -4
 liburcu_la_LIBADD = liburcu-common.la
 
-liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT)
+liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT)
 liburcu_qsbr_la_LIBADD = liburcu-common.la
 
-liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT)
 liburcu_mb_la_CFLAGS = -DRCU_MB
 liburcu_mb_la_LIBADD = liburcu-common.la
 
-liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_signal.lds $(COMPAT)
 liburcu_signal_la_CFLAGS = -DRCU_SIGNAL
 liburcu_signal_la_LIBADD = liburcu-common.la
 
-liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT)
+liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c compat_rcu_gp_bp.lds $(COMPAT)
 liburcu_bp_la_LIBADD = liburcu-common.la
 
 liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \
compat_rcu_gp.lds new file +25 -0
new file mode 100644
index 0000000..506a4ae
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp.lds
+ *
+ * Linker Script for compatibility of liburcu
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_memb = rcu_gp_memb;
+rcu_gp_futex_memb = rcu_gp_memb + 8;
+
compat_rcu_gp_bp.lds new file +24 -0
new file mode 100644
index 0000000..6ab7dfa
@@ -0,0 +1,24 @@
+/*
+ * compat_rcu_gp_bp.lds
+ *
+ * Linker Script for compatibility of liburcu-bp
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_bp = rcu_gp_bp;
+
compat_rcu_gp_mb.lds new file +25 -0
new file mode 100644
index 0000000..8313588
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_mb.lds
+ *
+ * Linker Script for compatibility of liburcu-mb
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_mb = rcu_gp_mb;
+rcu_gp_futex_mb = rcu_gp_mb + 8;
+
compat_rcu_gp_qsbr.lds new file +25 -0
new file mode 100644
index 0000000..d27e163
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_qsbr.lds
+ *
+ * Linker Script for compatibility of liburcu-qsbr
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_qsbr = rcu_gp_qsbr;
+rcu_gp_futex_qsbr = rcu_gp_qsbr + 8;
+
compat_rcu_gp_signal.lds new file +25 -0
new file mode 100644
index 0000000..f0a447d
@@ -0,0 +1,25 @@
+/*
+ * compat_rcu_gp_signal.lds
+ *
+ * Linker Script for compatibility of liburcu-signal
+ *
+ * Copyright (c) 2012 Lai Jiangshan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+rcu_gp_ctr_signal = rcu_gp_signal;
+rcu_gp_futex_signal = rcu_gp_signal + 8;
+
urcu-bp.c +4 -10
index f99c0e5..2a2455b 100644
@@ -103,6 +103,8 @@ void *mremap_wrapper(void *old_address, size_t old_size,
 void __attribute__((destructor)) rcu_bp_exit(void);
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
+/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 #ifdef DEBUG_YIELD
 unsigned int rcu_yield_active;
@@ -110,14 +112,6 @@ DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);
 #endif
 
 /*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
-
-/*
  * Pointer to registry elements. Written to only by each individual reader. Read
  * by both the reader and the writers.
  */
@@ -174,7 +168,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
@@ -250,7 +244,7 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
 	 * Must commit qparity update to memory before waiting for other parity
urcu-qsbr.c +15 -20
index 3c2c65d..5e2510d 100644
@@ -53,13 +53,8 @@
 void __attribute__((destructor)) rcu_exit(void);
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- */
-unsigned long rcu_gp_ctr = RCU_GP_ONLINE;
+/* Init the ctr as RCU_GP_ONLINE, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { { .ctr = RCU_GP_ONLINE } };
 
 /*
  * Active attempts to check for reader Q.S. before calling futex().
@@ -118,8 +113,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	cmm_smp_rmb();
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_noasync(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_noasync(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -133,12 +128,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (offline), or for them to observe the
-	 * current rcu_gp_ctr value.
+	 * current rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_set(&rcu_gp_futex, -1);
+			uatomic_set(&rcu_gp.futex, -1);
 			/*
 			 * Write futex before write waiting (the other side
 			 * reads them in the opposite order).
@@ -177,7 +172,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				cmm_smp_mb();
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -253,11 +248,11 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity
-	 * before committing next rcu_gp_ctr update to memory. Failure
+	 * before committing next rcu_gp.ctr update to memory. Failure
 	 * to do so could result in the writer waiting forever while new
 	 * readers are always accessing data (no progress).  Enforce
 	 * compiler-order of load URCU_TLS(rcu_reader).ctr before store
-	 * to rcu_gp_ctr.
+	 * to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -269,13 +264,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for
+	 * Must commit rcu_gp.ctr update to memory before waiting for
 	 * quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data
-	 * (no progress). Enforce compiler-order of store to rcu_gp_ctr
+	 * (no progress). Enforce compiler-order of store to rcu_gp.ctr
 	 * before load URCU_TLS(rcu_reader).ctr.
 	 */
 	cmm_barrier();
@@ -353,13 +348,13 @@ void synchronize_rcu(void)
 		goto out;
 
 	/* Increment current G.P. */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr + RCU_GP_CTR);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for
+	 * Must commit rcu_gp.ctr update to memory before waiting for
 	 * quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data
-	 * (no progress). Enforce compiler-order of store to rcu_gp_ctr
+	 * (no progress). Enforce compiler-order of store to rcu_gp.ctr
 	 * before load URCU_TLS(rcu_reader).ctr.
 	 */
 	cmm_barrier();
urcu.c +13 -21
index 15def09..782c42e 100644
@@ -83,16 +83,8 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+/* Init the ctr as RCU_GP_COUNT, to accelerate the reader fast path. */
+struct urcu_gp rcu_gp = { { .ctr = RCU_GP_COUNT } };
 
 /*
  * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +209,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	smp_mb_master(RCU_MB_GROUP);
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -232,12 +224,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_dec(&rcu_gp_futex);
+			uatomic_dec(&rcu_gp.futex);
 			/* Write futex before read reader_gp */
 			smp_mb_master(RCU_MB_GROUP);
 		}
@@ -270,7 +262,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -289,7 +281,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -357,10 +349,10 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity before
-	 * committing next rcu_gp_ctr update to memory. Failure to do so could
+	 * committing next rcu_gp.ctr update to memory. Failure to do so could
 	 * result in the writer waiting forever while new readers are always
 	 * accessing data (no progress).  Enforce compiler-order of load
-	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -372,13 +364,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for quiescent
+	 * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 	 * state. Failure to do so could result in the writer waiting forever
 	 * while new readers are always accessing data (no progress). Enforce
-	 * compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
+	 * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 	 */
 	cmm_barrier();
 
urcu/map/urcu-bp.h +1 -2
index 63582fd..0464d3b 100644
@@ -44,8 +44,7 @@
 #define rcu_exit			rcu_exit_bp
 #define synchronize_rcu			synchronize_rcu_bp
 #define rcu_reader			rcu_reader_bp
-#define rcu_gp_ctr			rcu_gp_ctr_bp
-#define rcu_gp_futex			rcu_gp_futex_bp	/* unused */
+#define rcu_gp				rcu_gp_bp
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_bp
 #define get_call_rcu_thread		get_call_rcu_thread_bp
urcu/map/urcu-qsbr.h +1 -2
index f0aecc9..2c22baf 100644
@@ -47,8 +47,7 @@
 #define rcu_exit			rcu_exit_qsbr
 #define synchronize_rcu			synchronize_rcu_qsbr
 #define rcu_reader			rcu_reader_qsbr
-#define rcu_gp_ctr			rcu_gp_ctr_qsbr
-#define rcu_gp_futex			rcu_gp_futex_qsbr
+#define rcu_gp				rcu_gp_qsbr
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_qsbr
 #define get_call_rcu_thread		get_call_rcu_thread_qsbr
urcu/map/urcu.h +3 -6
index 759cb41..5ef3d4f 100644
@@ -75,8 +75,7 @@
 #define rcu_exit			rcu_exit_memb
 #define synchronize_rcu			synchronize_rcu_memb
 #define rcu_reader			rcu_reader_memb
-#define rcu_gp_ctr			rcu_gp_ctr_memb
-#define rcu_gp_futex			rcu_gp_futex_memb
+#define rcu_gp				rcu_gp_memb
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_memb
 #define get_call_rcu_thread		get_call_rcu_thread_memb
@@ -121,8 +120,7 @@
 #define rcu_exit			rcu_exit_sig
 #define synchronize_rcu			synchronize_rcu_sig
 #define rcu_reader			rcu_reader_sig
-#define rcu_gp_ctr			rcu_gp_ctr_sig
-#define rcu_gp_futex			rcu_gp_futex_sig
+#define rcu_gp				rcu_gp_sig
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_sig
 #define get_call_rcu_thread		get_call_rcu_thread_sig
@@ -164,8 +162,7 @@
 #define rcu_exit			rcu_exit_mb
 #define synchronize_rcu			synchronize_rcu_mb
 #define rcu_reader			rcu_reader_mb
-#define rcu_gp_ctr			rcu_gp_ctr_mb
-#define rcu_gp_futex			rcu_gp_futex_mb
+#define rcu_gp				rcu_gp_mb
 
 #define get_cpu_call_rcu_data		get_cpu_call_rcu_data_mb
 #define get_call_rcu_thread		get_call_rcu_thread_mb
urcu/static/urcu-bp.h +14 -9
index c7f5326..a773a01 100644
@@ -130,12 +130,17 @@ static inline void rcu_debug_yield_init(void)
  */
 extern void rcu_bp_register(void);
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	/*
+	 * Global quiescent period counter with low-order bits unused.
+	 * Contains the current RCU_GP_CTR_PHASE.
+	 * Written to only by writer with mutex taken.
+	 * Read by both writer and readers.
+	 */
+	unsigned long ctr;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -166,13 +171,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -181,7 +186,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		cmm_smp_mb();
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp + RCU_GP_COUNT);
urcu/static/urcu-qsbr.h +0 -6
index f6e5580..d7a1e1c 100644
@@ -119,12 +119,23 @@ static inline void rcu_debug_yield_init(void)
 #define RCU_GP_ONLINE		(1UL << 0)
 #define RCU_GP_CTR		(1UL << 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;

urcu/static/urcu-qsbr.h static inline void rcu_debug_yield_init(void) · line 131

Quoted 10 lines
urcu/static/urcu-qsbr.h +10 -0
+struct urcu_gp {
+	union {
+		/*
+		 * Global quiescent period counter with low-order bits unused.
+		 * Using a int rather than a char to eliminate false register
+		 * dependencies causing stalls on some architectures.
+		 */
+		unsigned long ctr;
+
+		/* ensure the futex is located at offset=8 */

The comment could probably be more verbose:

/*
* Ensure futex field is located at offset=8 for backward
* compatibility linker script.
*/

urcu/static/urcu-qsbr.h static inline void rcu_debug_yield_init(void) · line 132

Quoted 1 line
urcu/static/urcu-qsbr.h +1 -0
+		int64_t __unused;

I would be tempted to do:

urcu-linker.h:

#define URCU_GP_FUTEX_OFFSET 8

and use a

                char __unused[URCU_GP_FUTEX_OFFSET];

in the structure (and use this macro in the linker scripts as well).

The advantage of using a char rather than a int64_t is that it won't get in the way of type pruning, and will allow more aliasing analysis to take place: AFAIK, adding a int64_t aliased with a unsigned long within a union requires that a compiler doing type pruning considers that those two types can alias pretty much anywhere in a file including our header.

Thanks,

Mathieu

Quoted 151 lines
urcu/static/urcu-qsbr.h +13 -9
+	};
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -137,8 +148,6 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
@@ -147,10 +156,10 @@ static inline void wake_up_gp(void)
 	if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
 		cmm_smp_mb();
-		if (uatomic_read(&rcu_gp_futex) != -1)
+		if (uatomic_read(&rcu_gp.futex) != -1)
 			return;
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_noasync(&rcu_gp_futex, FUTEX_WAKE, 1,
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_noasync(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -162,7 +171,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!v)
 		return RCU_READER_INACTIVE;
-	if (v == rcu_gp_ctr)
+	if (v == rcu_gp.ctr)
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
@@ -216,7 +225,7 @@ static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
  * to be invoked directly from non-LGPL code.
  *
  * We skip the memory barriers and gp store if our local ctr already
- * matches the global rcu_gp_ctr value: this is OK because a prior
+ * matches the global rcu_gp.ctr value: this is OK because a prior
  * _rcu_quiescent_state() or _rcu_thread_online() already updated it
  * within our thread, so we have no quiescent state to report.
  */
@@ -224,7 +233,7 @@ static inline void _rcu_quiescent_state(void)
 {
 	unsigned long gp_ctr;
 
-	if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp_ctr)) == URCU_TLS(rcu_reader).ctr)
+	if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp.ctr)) == URCU_TLS(rcu_reader).ctr)
 		return;
 	_rcu_quiescent_state_update_and_wakeup(gp_ctr);
 }
@@ -257,7 +266,7 @@ static inline void _rcu_thread_offline(void)
 static inline void _rcu_thread_online(void)
 {
 	cmm_barrier();	/* Ensure the compiler does not reorder us with mutex */
-	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
+	_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp.ctr));
 	cmm_smp_mb();
 }
 
urcu/static/urcu.h +24 -14
index 973826a..d23e673 100644
@@ -213,12 +213,24 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE	(1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK	(RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	union {
+		/*
+		 * Global grace period counter.
+		 * Contains the current RCU_GP_CTR_PHASE.
+		 * Written to only by writer with mutex taken.
+		 * Read by both writer and readers.
+		 */
+		unsigned long ctr;
+
+		/* ensure the futex is located at offset=8 */
+		int64_t __unused;
+	};
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +243,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
 static inline void wake_up_gp(void)
 {
-	if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+	if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -256,13 +266,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +281,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		smp_mb_slave(RCU_MB_GROUP);
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
Signature · 1 line
1.7.4.4
Signature · 4 lines
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20121211123802.GA4356@Krystal

#22 lore raw
Quoted 66 lines · Mathieu Desnoyers
Quoted 65 lines · * Lai Jiangshan ([email protected]) wrote:

From 66f50786b9d5f1bcf2ad56e3a9ab905005c6cd90 Mon Sep 17 00:00:00 2001
From: Lai Jiangshan <[email protected]>
Date: Tue, 11 Dec 2012 14:24:44 +0800
Subject: [PATCH] urcu: avoid false sharing for rcu_gp_ctr

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---
 Makefile.am              |   10 +++++-----
 compat_rcu_gp.lds        |   25 +++++++++++++++++++++++++
 compat_rcu_gp_bp.lds     |   24 ++++++++++++++++++++++++
 compat_rcu_gp_mb.lds     |   25 +++++++++++++++++++++++++
 compat_rcu_gp_qsbr.lds   |   25 +++++++++++++++++++++++++
 compat_rcu_gp_signal.lds |   25 +++++++++++++++++++++++++
 urcu-bp.c                |   14 ++++----------
 urcu-qsbr.c              |   35 +++++++++++++++--------------------
 urcu.c                   |   34 +++++++++++++---------------------
 urcu/map/urcu-bp.h       |    3 +--
 urcu/map/urcu-qsbr.h     |    3 +--
 urcu/map/urcu.h          |    9 +++------
 urcu/static/urcu-bp.h    |   23 ++++++++++++++---------
 urcu/static/urcu-qsbr.h  |   39 ++++++++++++++++++++++++---------------
 urcu/static/urcu.h       |   38 ++++++++++++++++++++++++--------------
 15 files changed, 228 insertions(+), 104 deletions(-)
 create mode 100644 compat_rcu_gp.lds
 create mode 100644 compat_rcu_gp_bp.lds
 create mode 100644 compat_rcu_gp_mb.lds
 create mode 100644 compat_rcu_gp_qsbr.lds
 create mode 100644 compat_rcu_gp_signal.lds
Makefile.am +0 -0
index 0a4d357..b2b7b6a 100644
@@ -56,21 +56,21 @@ lib_LTLIBRARIES = liburcu-common.la \
 #
 liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
 
Quoted 5 lines

Makefile.am lib_LTLIBRARIES = liburcu-common.la \ · line 59

Quoted 2 lines
Makefile.am +1 -1
-liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp.lds $(COMPAT)

Hrm, I tried your patch with "make V=1", and have not found any use of those "lds" files. I'm not sure adding them as SOURCES is enough.

Use liburcu_mb_la_LDFLAGS = -Xlinker compat_rcu_gp_mb.lds instead!

Result(nm .libs/liburcu-mb.so):
00000000002046e0 D rcu_flavor_mb
0000000000204a80 D rcu_gp_ctr_mb
0000000000204a88 A rcu_gp_futex_mb
0000000000204c60 b rcu_gp_lock
0000000000204a80 D rcu_gp_mb

The type "A" makes me nervous, Do we expect this?

And a new problem:
liburcu.so is RCU_MB actually in my system, but compat_rcu_gp.lds uses rcu_gp_memb(RCU_MEMBARRIER), it can't work in my system, how to fix it?

Quoted 11 lines
Quoted 11 lines
Makefile.am +2 -2
 liburcu_la_LIBADD = liburcu-common.la
 
-liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT)
+liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c compat_rcu_gp_qsbr.lds $(COMPAT)
 liburcu_qsbr_la_LIBADD = liburcu-common.la
 
-liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c compat_rcu_gp_mb.lds $(COMPAT)
 liburcu_mb_la_CFLAGS = -DRCU_MB
 liburcu_mb_la_LIBADD = liburcu-common.la
#23 lore raw
In reply to a message not in the archive

Since we are bumping the urcu soname version number to 2.0.0 for the upcoming urcu 0.8 anyway, it's time to merge this patch.

Thanks!

Mathieu

Quoted 204 lines · * Lai Jiangshan ([email protected]) wrote:

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---

urcu.c +12 -21
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	smp_mb_master(RCU_MB_GROUP);
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_dec(&rcu_gp_futex);
+			uatomic_dec(&rcu_gp.futex);
 			/* Write futex before read reader_gp */
 			smp_mb_master(RCU_MB_GROUP);
 		}
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -357,10 +348,10 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity before
-	 * committing next rcu_gp_ctr update to memory. Failure to do so could
+	 * committing next rcu_gp.ctr update to memory. Failure to do so could
 	 * result in the writer waiting forever while new readers are always
 	 * accessing data (no progress).  Enforce compiler-order of load
-	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -372,13 +363,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for quiescent
+	 * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 	 * state. Failure to do so could result in the writer waiting forever
 	 * while new readers are always accessing data (no progress). Enforce
-	 * compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
+	 * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 	 */
 	cmm_barrier();
 
urcu/static/urcu.h +20 -14
index 973826a..0dd733e 100644
@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE	(1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK	(RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	/*
+	 * Global grace period counter.
+	 * Contains the current RCU_GP_CTR_PHASE.
+	 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
+	 * Written to only by writer with mutex taken.
+	 * Read by both writer and readers.
+	 */
+	unsigned long ctr;
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +239,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
 static inline void wake_up_gp(void)
 {
-	if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+	if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		smp_mb_slave(RCU_MB_GROUP);
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20130506124433.GB12699@Krystal

#25 lore raw

Hi, Mathieu,

There is a big compatible problem in URCU which should be fix in next round.

LB: liburcu built on the system which has sys_membarrier().
LU: liburcu built on the system which does NOT have sys_membarrier().

LBM: liburcu-mb ....
LUM: liburcu-mb ...

AB: application(-lliburcu) built on the system which has sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT have sys_membarrier().

ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...

AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinations

I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?

could you check it?

How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to URCU_MB.

Thanks,
Lai

Quoted 213 lines · Mathieu Desnoyers

Since we are bumping the urcu soname version number to 2.0.0 for the upcoming urcu 0.8 anyway, it's time to merge this patch.

Thanks!

Mathieu

Quoted 204 lines · * Lai Jiangshan ([email protected]) wrote:

@rcu_gp_ctr and @registry share the same cache line, it causes false sharing and slowdown both of the read site and update site.

Fix: Use different cache line for them.

Although rcu_gp_futex is updated less than rcu_gp_ctr, but
they always be accessed at almost the same time, so we also move rcu_gp_futex to the cacheline of rcu_gp_ctr to reduce the cacheline-usage or cache-missing of read site.

test: (4X6=24 CPUs)

Before patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 2100285330 nr_writes 3390219 nr_ops 2103675549
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1619868562 nr_writes 3529478 nr_ops 1623398040
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 1949067038 nr_writes 3469334 nr_ops 1952536372

after patch:

[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3380191848 nr_writes 4903248 nr_ops 3385095096
[root@localhost userspace-rcu]# ./tests/test_urcu_mb 20 1 20
SUMMARY ./tests/test_urcu_mb testdur 20 nr_readers 20 rdur 0 wdur 0 nr_writers 1 wdelay 0 nr_reads 3397637486 nr_writes 4129809 nr_ops 3401767295

---

urcu.c +12 -21
index 15def09..436d71c 100644
@@ -83,16 +83,7 @@ void __attribute__((destructor)) rcu_exit(void);
 #endif
 
 static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-
-int32_t rcu_gp_futex;
-
-/*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken. Read by both writer and readers.
- */
-unsigned long rcu_gp_ctr = RCU_GP_COUNT;
+struct urcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
 
 /*
  * Written to only by each individual reader. Read by both the reader and the
@@ -217,8 +208,8 @@ static void wait_gp(void)
 {
 	/* Read reader_gp before read futex */
 	smp_mb_master(RCU_MB_GROUP);
-	if (uatomic_read(&rcu_gp_futex) == -1)
-		futex_async(&rcu_gp_futex, FUTEX_WAIT, -1,
+	if (uatomic_read(&rcu_gp.futex) == -1)
+		futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
 		      NULL, NULL, 0);
 }
 
@@ -232,12 +223,12 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 	/*
 	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
 	 * indicate quiescence (not nested), or observe the current
-	 * rcu_gp_ctr value.
+	 * rcu_gp.ctr value.
 	 */
 	for (;;) {
 		wait_loops++;
 		if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
-			uatomic_dec(&rcu_gp_futex);
+			uatomic_dec(&rcu_gp.futex);
 			/* Write futex before read reader_gp */
 			smp_mb_master(RCU_MB_GROUP);
 		}
@@ -270,7 +261,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -289,7 +280,7 @@ static void wait_for_readers(struct cds_list_head *input_readers,
 			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) {
 				/* Read reader_gp before write futex */
 				smp_mb_master(RCU_MB_GROUP);
-				uatomic_set(&rcu_gp_futex, 0);
+				uatomic_set(&rcu_gp.futex, 0);
 			}
 			break;
 		} else {
@@ -357,10 +348,10 @@ void synchronize_rcu(void)
 
 	/*
 	 * Must finish waiting for quiescent state for original parity before
-	 * committing next rcu_gp_ctr update to memory. Failure to do so could
+	 * committing next rcu_gp.ctr update to memory. Failure to do so could
 	 * result in the writer waiting forever while new readers are always
 	 * accessing data (no progress).  Enforce compiler-order of load
-	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp_ctr.
+	 * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
 	 */
 	cmm_barrier();
 
@@ -372,13 +363,13 @@ void synchronize_rcu(void)
 	cmm_smp_mb();
 
 	/* Switch parity: 0 -> 1, 1 -> 0 */
-	CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
+	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
 
 	/*
-	 * Must commit rcu_gp_ctr update to memory before waiting for quiescent
+	 * Must commit rcu_gp.ctr update to memory before waiting for quiescent
 	 * state. Failure to do so could result in the writer waiting forever
 	 * while new readers are always accessing data (no progress). Enforce
-	 * compiler-order of store to rcu_gp_ctr before load rcu_reader ctr.
+	 * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
 	 */
 	cmm_barrier();
 
urcu/static/urcu.h +20 -14
index 973826a..0dd733e 100644
@@ -213,12 +213,20 @@ static inline void smp_mb_slave(int group)
 #define RCU_GP_CTR_PHASE	(1UL << (sizeof(unsigned long) << 2))
 #define RCU_GP_CTR_NEST_MASK	(RCU_GP_CTR_PHASE - 1)
 
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
+struct urcu_gp {
+	/*
+	 * Global grace period counter.
+	 * Contains the current RCU_GP_CTR_PHASE.
+	 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
+	 * Written to only by writer with mutex taken.
+	 * Read by both writer and readers.
+	 */
+	unsigned long ctr;
+
+	int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct urcu_gp rcu_gp;
 
 struct rcu_reader {
 	/* Data used by both reader and synchronize_rcu() */
@@ -231,16 +239,14 @@ struct rcu_reader {
 
 extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
 
-extern int32_t rcu_gp_futex;
-
 /*
  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  */
 static inline void wake_up_gp(void)
 {
-	if (caa_unlikely(uatomic_read(&rcu_gp_futex) == -1)) {
-		uatomic_set(&rcu_gp_futex, 0);
-		futex_async(&rcu_gp_futex, FUTEX_WAKE, 1,
+	if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+		uatomic_set(&rcu_gp.futex, 0);
+		futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
 		      NULL, NULL, 0);
 	}
 }
@@ -256,13 +262,13 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 	v = CMM_LOAD_SHARED(*ctr);
 	if (!(v & RCU_GP_CTR_NEST_MASK))
 		return RCU_READER_INACTIVE;
-	if (!((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE))
+	if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
 		return RCU_READER_ACTIVE_CURRENT;
 	return RCU_READER_ACTIVE_OLD;
 }
 
 /*
- * Helper for _rcu_read_lock().  The format of rcu_gp_ctr (as well as
+ * Helper for _rcu_read_lock().  The format of rcu_gp.ctr (as well as
  * the per-thread rcu_reader.ctr) has the upper bits containing a count of
  * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
  * or RCU_GP_CTR_PHASE.  The smp_mb_slave() ensures that the accesses in
@@ -271,7 +277,7 @@ static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
 static inline void _rcu_read_lock_update(unsigned long tmp)
 {
 	if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
-		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
 		smp_mb_slave(RCU_MB_GROUP);
 	} else
 		_CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);

A message that is not in the archive (sent off-list or not yet received): [email protected]

#27 lore raw
Quoted 19 lines · * Lai Jiangshan ([email protected]) wrote:

Hi, Mathieu,

There is a big compatible problem in URCU which should be fix in next round.

LB: liburcu built on the system which has sys_membarrier().
LU: liburcu built on the system which does NOT have sys_membarrier().

LBM: liburcu-mb ....
LUM: liburcu-mb ...

AB: application(-lliburcu) built on the system which has sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT have sys_membarrier().

ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...

AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinations

Quoted 7 lines

I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?

could you check it?

How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to URCU_MB.

Hi Lai,

Good catch !

Would the following patch work for you ? It introduces RCU_MEMBARRIER_MB for implementation fallback, but keeps RCU_MEMBARRIER defined and used for the RCU flavor mapping file.

Thoughts ?

Thanks,

Mathieu

---

urcu.c +5 -5
index b3f94da..352d63f 100644
@@ -62,14 +62,14 @@
  */
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static int init_done;
 int rcu_has_sys_membarrier;
 
 void __attribute__((constructor)) rcu_init(void);
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 }
@@ -135,7 +135,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 		urcu_die(ret);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
@@ -145,7 +145,7 @@ static void smp_mb_master(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	cmm_smp_mb();
@@ -444,7 +444,7 @@ void rcu_unregister_thread(void)
 	mutex_unlock(&rcu_gp_lock);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 	if (init_done)
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +7 -9
index bee97ee..be3d20c 100644
@@ -52,15 +52,14 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
+ * RCU_MEMBARRIER_MB otherwise.
  */
 #if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
+#define RCU_MEMBARRIER_MB
 #endif
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 #include <syscall.h>
 
 /* If the headers do not support SYS_membarrier, statically use RCU_MB */
@@ -70,8 +69,7 @@ extern "C" {
 # define MEMBARRIER_QUERY		(1 << 16)
 # define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
 #else
-# undef RCU_MEMBARRIER
-# define RCU_MB
+# define RCU_MEMBARRIER_MB
 #endif
 #endif
 
@@ -178,7 +176,7 @@ static inline void rcu_debug_yield_init(void)
 #define MB_GROUP_ALL		0
 #define RCU_MB_GROUP		MB_GROUP_ALL
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 extern int rcu_has_sys_membarrier;
 
 static inline void smp_mb_slave(int group)
@@ -190,7 +188,7 @@ static inline void smp_mb_slave(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static inline void smp_mb_slave(int group)
 {
 	cmm_smp_mb();
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20130508135343.GA8230@Krystal

#29 lore raw
Quoted 48 lines · Mathieu Desnoyers
Quoted 19 lines · * Lai Jiangshan ([email protected]) wrote:

Hi, Mathieu,

There is a big compatible problem in URCU which should be fix in next round.

LB: liburcu built on the system which has sys_membarrier().
LU: liburcu built on the system which does NOT have sys_membarrier().

LBM: liburcu-mb ....
LUM: liburcu-mb ...

AB: application(-lliburcu) built on the system which has sys_membarrier().
AU: application(-lliburcu) built on the system which does NOT have sys_membarrier().

ABM application(-lliburcu-mb) ...
AUM application(-lliburcu-mb) ...

AB/AU + LB/LU: 4 combinations
ABM/AUM + LBM/LUM: 4 combinations

Quoted 7 lines

I remember some of the 8 combinations can't works due to symbols are miss match. only LU+AB and LB+AU ?

could you check it?

How to fix it: In LU and AU, keep all the symbol name/ABI as LA and AB, but only the behaviors falls back to URCU_MB.

Hi Lai,

Good catch !

Would the following patch work for you ? It introduces RCU_MEMBARRIER_MB for implementation fallback, but keeps RCU_MEMBARRIER defined and used for the RCU flavor mapping file.

Thoughts ?

Thanks,

Mathieu

---

urcu.c +0 -0
index b3f94da..352d63f 100644

urcu.c line 67

Quoted 7 lines
urcu.c +1 -1
@@ -62,14 +62,14 @@
  */
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static int init_done;
 int rcu_has_sys_membarrier;

AB will use rcu_has_sys_membarrier, but LU don't have it.

RCU_MEMBARRIER_MB still introduces different ABI.

Quoted 134 lines
urcu.c +4 -4
 void __attribute__((constructor)) rcu_init(void);
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 }
@@ -135,7 +135,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
                urcu_die(ret);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
        if (caa_likely(rcu_has_sys_membarrier))
@@ -145,7 +145,7 @@ static void smp_mb_master(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
        cmm_smp_mb();
@@ -444,7 +444,7 @@ void rcu_unregister_thread(void)
        mutex_unlock(&rcu_gp_lock);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
        if (init_done)
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED          (1 << 0)
-# define MEMBARRIER_DELAYED            (1 << 1)
-# define MEMBARRIER_QUERY              (1 << 16)
-# define membarrier(...)               syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock                  rcu_read_lock_memb
urcu/static/urcu.h +7 -10
index bee97ee..be3d20c 100644
@@ -52,15 +52,14 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
+ * RCU_MEMBARRIER_MB otherwise.
  */
 #if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
+#define RCU_MEMBARRIER_MB
 #endif
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 #include <syscall.h>
 
 /* If the headers do not support SYS_membarrier, statically use RCU_MB */
@@ -70,8 +69,7 @@ extern "C" {
 # define MEMBARRIER_QUERY              (1 << 16)
 # define membarrier(...)               syscall(SYS_membarrier, __VA_ARGS__)
 #else
-# undef RCU_MEMBARRIER
-# define RCU_MB
+# define RCU_MEMBARRIER_MB
 #endif
 #endif
 
@@ -178,7 +176,7 @@ static inline void rcu_debug_yield_init(void)
 #define MB_GROUP_ALL           0
 #define RCU_MB_GROUP           MB_GROUP_ALL
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 extern int rcu_has_sys_membarrier;
 
 static inline void smp_mb_slave(int group)
@@ -190,7 +188,7 @@ static inline void smp_mb_slave(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static inline void smp_mb_slave(int group)
 {
        cmm_smp_mb();
 
--

Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): CACvQF521cSR0_5K+aC79A+s9APypQOS8POEoP2cDRguguw6P9A@mail.gmail.com

#31 lore raw

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 3 lines

AB will use rcu_has_sys_membarrier, but LU don't have it.

RCU_MEMBARRIER_MB still introduces different ABI.

Yes, you're right. How about the following then ?

---

urcu.c +8 -5
index b3f94da..a7d9294 100644
@@ -63,13 +63,16 @@
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
 #ifdef RCU_MEMBARRIER
-static int init_done;
 int rcu_has_sys_membarrier;
+#endif
+
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
+static int init_done;
 
 void __attribute__((constructor)) rcu_init(void);
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 }
@@ -135,7 +138,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 		urcu_die(ret);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
@@ -145,7 +148,7 @@ static void smp_mb_master(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	cmm_smp_mb();
@@ -444,7 +447,7 @@ void rcu_unregister_thread(void)
 	mutex_unlock(&rcu_gp_lock);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 	if (init_done)
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +5 -7
index bee97ee..633a46f 100644
@@ -52,15 +52,14 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
+ * RCU_MEMBARRIER_MB otherwise.
  */
 #if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
+#define RCU_MEMBARRIER_MB
 #endif
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 #include <syscall.h>
 
 /* If the headers do not support SYS_membarrier, statically use RCU_MB */
@@ -70,8 +69,7 @@ extern "C" {
 # define MEMBARRIER_QUERY		(1 << 16)
 # define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
 #else
-# undef RCU_MEMBARRIER
-# define RCU_MB
+# define RCU_MEMBARRIER_MB
 #endif
 #endif
 
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20130508181936.GA9919@Krystal

#33 lore raw
Quoted 121 lines · Mathieu Desnoyers

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 3 lines

AB will use rcu_has_sys_membarrier, but LU don't have it.

RCU_MEMBARRIER_MB still introduces different ABI.

Yes, you're right. How about the following then ?

---

urcu.c +8 -5
index b3f94da..a7d9294 100644
@@ -63,13 +63,16 @@
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
 #ifdef RCU_MEMBARRIER
-static int init_done;
 int rcu_has_sys_membarrier;
+#endif
+
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
+static int init_done;
 
 void __attribute__((constructor)) rcu_init(void);
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 }
@@ -135,7 +138,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 		urcu_die(ret);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
@@ -145,7 +148,7 @@ static void smp_mb_master(int group)
 }
 #endif
 
-#ifdef RCU_MB
+#if defined(RCU_MB) || defined(RCU_MEMBARRIER_MB)
 static void smp_mb_master(int group)
 {
 	cmm_smp_mb();
@@ -444,7 +447,7 @@ void rcu_unregister_thread(void)
 	mutex_unlock(&rcu_gp_lock);
 }
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 void rcu_init(void)
 {
 	if (init_done)
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +4 -5
index bee97ee..633a46f 100644
@@ -52,15 +52,14 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
+ * RCU_MEMBARRIER_MB otherwise.
  */
 #if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
+#define RCU_MEMBARRIER_MB
 #endif
 
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && !defined(RCU_MEMBARRIER_MB)
 #include <syscall.h>
 
 /* If the headers do not support SYS_membarrier, statically use RCU_MB */

urcu/static/urcu.h extern "C" { · line 75

Quoted 9 lines
urcu/static/urcu.h +1 -2
@@ -70,8 +69,7 @@ extern "C" {
 # define MEMBARRIER_QUERY		(1 << 16)
 # define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
 #else
-# undef RCU_MEMBARRIER
-# define RCU_MB
+# define RCU_MEMBARRIER_MB
 #endif
 #endif
 

How about we also define membarrier(...) which just returns -ENOSYS when the system don't have SYS_membarrier?

A message that is not in the archive (sent off-list or not yet received): [email protected]

#35 lore raw

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 2 lines

How about we also define membarrier(...) which just returns -ENOSYS when the system don't have SYS_membarrier?

Indeed, this vastly simplifies the patch:

Thoughts ?

---

urcu.c +1 -1
index b3f94da..763b40c 100644
@@ -139,7 +139,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
-		membarrier(MEMBARRIER_EXPEDITED);
+		(void) membarrier(MEMBARRIER_EXPEDITED);
 	else
 		cmm_smp_mb();
 }
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +10 -15
index bee97ee..0b03a50 100644
@@ -33,6 +33,7 @@
 #include <pthread.h>
 #include <unistd.h>
 #include <stdint.h>
+#include <error.h>
 
 #include <urcu/compiler.h>
 #include <urcu/arch.h>
@@ -52,29 +53,23 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux.
  */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && defined(__linux__)
 #include <syscall.h>
+#endif
 
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
+/* If the headers do not support SYS_membarrier, fall back on RCU_MB */
 #ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
 # define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
 #else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
+# define membarrier(...)		-ENOSYS
 #endif
 
+#define MEMBARRIER_EXPEDITED		(1 << 0)
+#define MEMBARRIER_DELAYED		(1 << 1)
+#define MEMBARRIER_QUERY		(1 << 16)
+
 /*
  * This code section can only be included in LGPL 2.1 compatible source code.
  * See below for the function call wrappers which can be used in code meant to
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20130509122748.GA23531@Krystal

#37 lore raw
Quoted 97 lines · Mathieu Desnoyers

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 2 lines

How about we also define membarrier(...) which just returns -ENOSYS when the system don't have SYS_membarrier?

Indeed, this vastly simplifies the patch:

Thoughts ?

---

urcu.c +1 -1
index b3f94da..763b40c 100644
@@ -139,7 +139,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 static void smp_mb_master(int group)
 {
        if (caa_likely(rcu_has_sys_membarrier))
-               membarrier(MEMBARRIER_EXPEDITED);
+               (void) membarrier(MEMBARRIER_EXPEDITED);
        else
                cmm_smp_mb();
 }
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED          (1 << 0)
-# define MEMBARRIER_DELAYED            (1 << 1)
-# define MEMBARRIER_QUERY              (1 << 16)
-# define membarrier(...)               syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock                  rcu_read_lock_memb
urcu/static/urcu.h +5 -12
index bee97ee..0b03a50 100644
@@ -33,6 +33,7 @@
 #include <pthread.h>
 #include <unistd.h>
 #include <stdint.h>
+#include <error.h>
 
 #include <urcu/compiler.h>
 #include <urcu/arch.h>
@@ -52,29 +53,23 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
+ * RCU_MEMBARRIER is only possibly available on Linux.
  */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
+#if defined(RCU_MEMBARRIER) && defined(__linux__)
 #include <syscall.h>
+#endif
 
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
+/* If the headers do not support SYS_membarrier, fall back on RCU_MB */
 #ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED          (1 << 0)
-# define MEMBARRIER_DELAYED            (1 << 1)
-# define MEMBARRIER_QUERY              (1 << 16)
 # define membarrier(...)               syscall(SYS_membarrier, __VA_ARGS__)
 #else

urcu/static/urcu.h extern "C" { · line 72

Quoted 10 lines
urcu/static/urcu.h +5 -3
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
+# define membarrier(...)               -ENOSYS
 #endif
 
+#define MEMBARRIER_EXPEDITED           (1 << 0)
+#define MEMBARRIER_DELAYED             (1 << 1)
+#define MEMBARRIER_QUERY               (1 << 16)
+

I think this code can be moved into urcu.c

Quoted 8 lines
urcu/static/urcu.h +0 -1
 /*
  * This code section can only be included in LGPL 2.1 compatible source code.
  * See below for the function call wrappers which can be used in code meant to
 
--

Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): CACvQF53K3Qwj4JO3y82yhxGqrBfPdkgH2fXcj5SbYO0OTD+uSg@mail.gmail.com

#39 lore raw

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 1 line

I think this code can be moved into urcu.c

good point! How about this ?

---

urcu.c +19 -1
index b3f94da..1d5c06f 100644
@@ -62,6 +62,24 @@
  */
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
+/*
+ * RCU_MEMBARRIER is only possibly available on Linux.
+ */
+#if defined(RCU_MEMBARRIER) && defined(__linux__)
+#include <syscall.h>
+#endif
+
+/* If the headers do not support SYS_membarrier, fall back on RCU_MB */
+#ifdef SYS_membarrier
+# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
+#else
+# define membarrier(...)		-ENOSYS
+#endif
+
+#define MEMBARRIER_EXPEDITED		(1 << 0)
+#define MEMBARRIER_DELAYED		(1 << 1)
+#define MEMBARRIER_QUERY		(1 << 16)
+
 #ifdef RCU_MEMBARRIER
 static int init_done;
 int rcu_has_sys_membarrier;
@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
-		membarrier(MEMBARRIER_EXPEDITED);
+		(void) membarrier(MEMBARRIER_EXPEDITED);
 	else
 		cmm_smp_mb();
 }
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +0 -24
index bee97ee..53d2610 100644
@@ -52,30 +52,6 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
-/*
  * This code section can only be included in LGPL 2.1 compatible source code.
  * See below for the function call wrappers which can be used in code meant to
  * be only linked with the Userspace RCU library. This comes with a small
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

A message that is not in the archive (sent off-list or not yet received): 20130509151018.GC27168@Krystal

#41 lore raw
Quoted 5 lines · Mathieu Desnoyers

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 1 line

I think this code can be moved into urcu.c

good point! How about this ?

Hi, I don't see any obvious bad thing, but I can't carefully check it currently, It seems OK for me.

Could you make it some progress.

Thanks,
Lai

Quoted 112 lines

---

urcu.c +19 -1
index b3f94da..1d5c06f 100644
@@ -62,6 +62,24 @@
  */
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
+/*
+ * RCU_MEMBARRIER is only possibly available on Linux.
+ */
+#if defined(RCU_MEMBARRIER) && defined(__linux__)
+#include <syscall.h>
+#endif
+
+/* If the headers do not support SYS_membarrier, fall back on RCU_MB */
+#ifdef SYS_membarrier
+# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
+#else
+# define membarrier(...)		-ENOSYS
+#endif
+
+#define MEMBARRIER_EXPEDITED		(1 << 0)
+#define MEMBARRIER_DELAYED		(1 << 1)
+#define MEMBARRIER_QUERY		(1 << 16)
+
 #ifdef RCU_MEMBARRIER
 static int init_done;
 int rcu_has_sys_membarrier;
@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
-		membarrier(MEMBARRIER_EXPEDITED);
+		(void) membarrier(MEMBARRIER_EXPEDITED);
 	else
 		cmm_smp_mb();
 }
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +0 -24
index bee97ee..53d2610 100644
@@ -52,30 +52,6 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
-/*
  * This code section can only be included in LGPL 2.1 compatible source code.
  * See below for the function call wrappers which can be used in code meant to
  * be only linked with the Userspace RCU library. This comes with a small

A message that is not in the archive (sent off-list or not yet received): [email protected]

#43 lore raw
Quoted 11 lines · * Lai Jiangshan ([email protected]) wrote:
Quoted 5 lines · Mathieu Desnoyers

* Lai Jiangshan ([email protected]) wrote:
[...]

Quoted 1 line

I think this code can be moved into urcu.c

good point! How about this ?

Hi, I don't see any obvious bad thing, but I can't carefully check it currently, It seems OK for me.

Could you make it some progress.

Thanks. Merged as:

commit 553b7eb9da3a223368abdf3f6a5ead35acf8a20b

Author: Mathieu Desnoyers <[email protected]>
Date:   Wed May 8 09:53:45 2013 -0400

Fix: membarrier fallback symbol conflict

Mathieu

Quoted 117 lines

Thanks,
Lai

Quoted 112 lines

---

urcu.c +19 -1
index b3f94da..1d5c06f 100644
@@ -62,6 +62,24 @@
  */
 #define RCU_QS_ACTIVE_ATTEMPTS 100
 
+/*
+ * RCU_MEMBARRIER is only possibly available on Linux.
+ */
+#if defined(RCU_MEMBARRIER) && defined(__linux__)
+#include <syscall.h>
+#endif
+
+/* If the headers do not support SYS_membarrier, fall back on RCU_MB */
+#ifdef SYS_membarrier
+# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
+#else
+# define membarrier(...)		-ENOSYS
+#endif
+
+#define MEMBARRIER_EXPEDITED		(1 << 0)
+#define MEMBARRIER_DELAYED		(1 << 1)
+#define MEMBARRIER_QUERY		(1 << 16)
+
 #ifdef RCU_MEMBARRIER
 static int init_done;
 int rcu_has_sys_membarrier;
@@ -139,7 +157,7 @@ static void mutex_unlock(pthread_mutex_t *mutex)
 static void smp_mb_master(int group)
 {
 	if (caa_likely(rcu_has_sys_membarrier))
-		membarrier(MEMBARRIER_EXPEDITED);
+		(void) membarrier(MEMBARRIER_EXPEDITED);
 	else
 		cmm_smp_mb();
 }
urcu/map/urcu.h +0 -25
index 217bff5..77b3721 100644
@@ -38,31 +38,6 @@
 #define RCU_MEMBARRIER
 #endif
 
-/*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to
- * RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
 #ifdef RCU_MEMBARRIER
 
 #define rcu_read_lock			rcu_read_lock_memb
urcu/static/urcu.h +0 -24
index bee97ee..53d2610 100644
@@ -52,30 +52,6 @@ extern "C" {
 #endif
 
 /*
- * RCU_MEMBARRIER is only possibly available on Linux. Fallback to RCU_MB
- * otherwise.
- */
-#if !defined(__linux__) && defined(RCU_MEMBARRIER)
-#undef RCU_MEMBARRIER
-#define RCU_MB
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED		(1 << 0)
-# define MEMBARRIER_DELAYED		(1 << 1)
-# define MEMBARRIER_QUERY		(1 << 16)
-# define membarrier(...)		syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
-/*
  * This code section can only be included in LGPL 2.1 compatible source code.
  * See below for the function call wrappers which can be used in code meant to
  * be only linked with the Userspace RCU library. This comes with a small
Signature · 3 lines
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com