introducing io_barrier

PATCH New Anton Blanchard 1 participant 1 message Dec 5, 2002

lore

Share & tools

lore

Anton Blanchard

author

2002-12-05 06:52 UTC #0 lore raw

PATCH 1/1 introducing io_barrier

Hi,

Currently we have three memory barrier primitives: mb, rmb and wmb. On ppc64 a full memory barrier (sync) can be very expensive, so the architecture has lighter weight barriers that can be used for rmb (lwsync) and wmb (eieio).

This works well until drivers start using these lighter weight barriers. eg from the e1000 driver:

        tx_desc->lower.data |= cpu_to_le32(E1000_TXD_CMD_EOP);

        /* Force memory writes to complete before letting h/w
         * know there are new descriptors to fetch.  (Only
         * applicable for weak-ordered memory model archs,
         * such as IA-64). */
        wmb();

        tx_ring->next_to_use = i;
        E1000_WRITE_REG(&adapter->hw, TDT, i);

wmb (eieio) guarantees to order stores to cacheable memory or stores to non cacheable memory (ie IO) but will not order between the two. So in the above example, E1000_WRITE_REG could complete before the store to tx_desc->lower.data.

My first response was to create io_barrier, io_barrier_read and io_barrier_write to match mb, rmb and wmb. However considering how badly we handle the current memory barriers I decided to restrict it to one function that will order between all cacheable and non cacheable memory - io_barrier. Architectures can optimise as they wish, Ive started them off as being a full memory barrier.

Thoughts?

Anton

===== include/asm-alpha/system.h 1.14 vs edited =====

include/asm-alpha/system.h +2 -0

@@ -155,6 +155,8 @@
 #define read_barrier_depends() \
 __asm__ __volatile__("mb": : :"memory")
 
+#define io_barrier()	mb()
+
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()

===== include/asm-arm/system.h 1.13 vs edited =====

include/asm-arm/system.h +1 -0

@@ -63,6 +63,7 @@
 #define rmb() mb()
 #define wmb() mb()
 #define read_barrier_depends() do { } while(0)
+#define io_barrier()	mb()
 #define set_mb(var, value)  do { var = value; mb(); } while (0)
 #define set_wmb(var, value) do { var = value; wmb(); } while (0)
 #define nop() __asm__ __volatile__("mov\tr0,r0\t@ nop\n\t");

===== include/asm-cris/system.h 1.7 vs edited =====

include/asm-cris/system.h +1 -0

@@ -151,6 +151,7 @@
 #define rmb() mb()
 #define wmb() mb()
 #define read_barrier_depends() do { } while(0)
+#define io_barrier()	mb()
 #define set_mb(var, value)  do { var = value; mb(); } while (0)
 #define set_wmb(var, value) do { var = value; wmb(); } while (0)

===== include/asm-i386/system.h 1.20 vs edited =====

include/asm-i386/system.h +1 -0

@@ -344,6 +344,7 @@
  **/
 
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #ifdef CONFIG_X86_OOSTORE
 #define wmb() 	__asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory")

===== include/asm-ia64/system.h 1.25 vs edited =====

include/asm-ia64/system.h +1 -0

@@ -87,6 +87,7 @@
 #define rmb()	mb()
 #define wmb()	mb()
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #ifdef CONFIG_SMP
 # define smp_mb()	mb()

===== include/asm-m68k/system.h 1.7 vs edited =====

include/asm-m68k/system.h +1 -0

@@ -74,6 +74,7 @@
 #define mb()		barrier()
 #define rmb()		barrier()
 #define wmb()		barrier()
+#define io_barrier()	mb()
 #define set_mb(var, value)    do { xchg(&var, value); } while (0)
 #define set_wmb(var, value)    do { var = value; wmb(); } while (0)

===== include/asm-m68knommu/system.h 1.1 vs edited =====

include/asm-m68knommu/system.h +1 -0

@@ -94,6 +94,7 @@
 #define mb()   asm volatile (""   : : :"memory")
 #define rmb()  asm volatile (""   : : :"memory")
 #define wmb()  asm volatile (""   : : :"memory")
+#define io_barrier()	mb()
 #define set_rmb(var, value)    do { xchg(&var, value); } while (0)
 #define set_mb(var, value)     set_rmb(var, value)
 #define set_wmb(var, value)    do { var = value; wmb(); } while (0)

===== include/asm-mips/system.h 1.5 vs edited =====

include/asm-mips/system.h +1 -0

@@ -147,6 +147,7 @@
 #define wmb()	wbflush()
 #define mb()	wbflush()
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #else /* CONFIG_CPU_HAS_WB  */

===== include/asm-mips64/system.h 1.4 vs edited =====

include/asm-mips64/system.h +1 -0

@@ -143,6 +143,7 @@
 #define rmb() mb()
 #define wmb() mb()
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()

===== include/asm-parisc/system.h 1.5 vs edited =====

include/asm-parisc/system.h +1 -0

@@ -132,6 +132,7 @@
 #define smp_wmb()	mb()
 #define smp_read_barrier_depends()	do { } while(0)
 #define read_barrier_depends()		do { } while(0)
+#define io_barrier()	mb()
 
 #define set_mb(var, value)		do { var = value; mb(); } while (0)
 #define set_wmb(var, value)		do { var = value; wmb(); } while (0)

===== include/asm-ppc/system.h 1.15 vs edited =====

include/asm-ppc/system.h +1 -0

@@ -33,6 +33,7 @@
 #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("eieio" : : : "memory")
 #define read_barrier_depends()  do { } while(0)
+#define io_barrier()	mb()
 
 #define set_mb(var, value)	do { var = value; mb(); } while (0)
 #define set_wmb(var, value)	do { var = value; wmb(); } while (0)

===== include/asm-ppc64/system.h 1.13 vs edited =====

include/asm-ppc64/system.h +1 -0

@@ -36,6 +36,7 @@
 #define rmb()  __asm__ __volatile__ ("lwsync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("eieio" : : : "memory")
 #define read_barrier_depends()  do { } while(0)
+#define io_barrier()	mb()
 
 #define set_mb(var, value)	do { var = value; mb(); } while (0)
 #define set_wmb(var, value)	do { var = value; wmb(); } while (0)

===== include/asm-s390/system.h 1.13 vs edited =====

include/asm-s390/system.h +1 -0

@@ -228,6 +228,7 @@
 #define rmb()   eieio()
 #define wmb()   eieio()
 #define read_barrier_depends() do { } while(0)
+#define io_barrier()	mb()
 #define smp_mb()       mb()
 #define smp_rmb()      rmb()
 #define smp_wmb()      wmb()

===== include/asm-s390x/system.h 1.12 vs edited =====

include/asm-s390x/system.h +1 -0

@@ -239,6 +239,7 @@
 #define rmb()   eieio()
 #define wmb()   eieio()
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 #define smp_mb()       mb()
 #define smp_rmb()      rmb()
 #define smp_wmb()      wmb()

===== include/asm-sh/system.h 1.5 vs edited =====

include/asm-sh/system.h +1 -0

@@ -90,6 +90,7 @@
 #define rmb()	mb()
 #define wmb()	__asm__ __volatile__ ("": : :"memory")
 #define read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()

===== include/asm-sparc/system.h 1.12 vs edited =====

include/asm-sparc/system.h +1 -0

@@ -297,6 +297,7 @@
 #define smp_rmb()	__asm__ __volatile__("":::"memory");
 #define smp_wmb()	__asm__ __volatile__("":::"memory");
 #define smp_read_barrier_depends()	do { } while(0)
+#define io_barrier()	mb()
 
 #define nop() __asm__ __volatile__ ("nop");

===== include/asm-sparc64/system.h 1.21 vs edited =====

include/asm-sparc64/system.h +1 -0

@@ -85,6 +85,7 @@
 #define rmb()		membar("#LoadLoad")
 #define wmb()		membar("#StoreStore")
 #define read_barrier_depends()		do { } while(0)
+#define io_barrier()	mb()
 #define set_mb(__var, __value) \
 	do { __var = __value; membar("#StoreLoad | #StoreStore"); } while(0)
 #define set_wmb(__var, __value) \

===== include/asm-v850/system.h 1.1 vs edited =====

include/asm-v850/system.h +1 -0

@@ -67,6 +67,7 @@
 #define mb()			__asm__ __volatile__ ("" ::: "memory")
 #define rmb()			mb ()
 #define wmb()			mb ()
+#define io_barrier()		mb()
 #define set_rmb(var, value)	do { xchg (&var, value); } while (0)
 #define set_mb(var, value)	set_rmb (var, value)
 #define set_wmb(var, value)	do { var = value; wmb (); } while (0)

===== include/asm-x86_64/system.h 1.8 vs edited =====

include/asm-x86_64/system.h +1 -0

@@ -233,6 +233,7 @@
 #define rmb()	asm volatile("lfence":::"memory")
 #define wmb()	asm volatile("sfence":::"memory")
 #define read_barrier_depends()	do {} while(0)
+#define io_barrier()	mb()
 #define set_mb(var, value) do { xchg(&var, value); } while (0)
 #define set_wmb(var, value) do { var = value; wmb(); } while (0)