Skip to contentSkip to conversation

sunrpc/xprtrdma: clean up workqueue usage

PATCH Applied Tejun Heo 3 participants 7 messages Oct 15, 2010 – Oct 21, 2010

Subsystems (3)
lore
Share & tools
lore

Tejun Heo

author
#0 lore raw

PATCH 1/1 sunrpc/xprtrdma: clean up workqueue usage

* Create and use svc_rdma_wq instead of using the system workqueue and flush_scheduled_work(). This workqueue is necessary to serve as flushing domain for rdma->sc_work which is used to destroy itself and thus can't be flushed explicitly.

* Replace cancel_delayed_work() + flush_scheduled_work() with cancel_delayed_work_sync().

* Implement synchronous connect in xprt_rdma_connect() using flush_delayed_work() on the rdma_connect work instead of using flush_scheduled_work().

This is to prepare for the deprecation and removal of flush_scheduled_work().

---
 net/sunrpc/xprtrdma/svc_rdma.c           |   11 ++++++++++-
 net/sunrpc/xprtrdma/svc_rdma_transport.c |    6 +++++-
 net/sunrpc/xprtrdma/transport.c          |    5 ++---
 3 files changed, 17 insertions(+), 5 deletions(-)
net/sunrpc/xprtrdma/svc_rdma.c +10 -1
Index: work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================
@@ -43,6 +43,7 @@
 #include <linux/slab.h>
 #include <linux/fs.h>
 #include <linux/sysctl.h>
+#include <linux/workqueue.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/sched.h>
 #include <linux/sunrpc/svc_rdma.h>
@@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
 struct kmem_cache *svc_rdma_map_cachep;
 struct kmem_cache *svc_rdma_ctxt_cachep;
 
+struct workqueue_struct *svc_rdma_wq;
+
 /*
  * This function implements reading and resetting an atomic_t stat
  * variable through read/write to a proc file. Any write to the file
@@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
 void svc_rdma_cleanup(void)
 {
 	dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
-	flush_scheduled_work();
+	destroy_workqueue(svc_rdma_wq);
 	if (svcrdma_table_header) {
 		unregister_sysctl_table(svcrdma_table_header);
 		svcrdma_table_header = NULL;
@@ -249,6 +252,11 @@ int svc_rdma_init(void)
 	dprintk("\tsq_depth         : %d\n",
 		svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
 	dprintk("\tmax_inline       : %d\n", svcrdma_max_req_size);
+
+	svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
+	if (!svc_rdma_wq)
+		return -ENOMEM;
+
 	if (!svcrdma_table_header)
 		svcrdma_table_header =
 			register_sysctl_table(svcrdma_root_table);
@@ -283,6 +291,7 @@ int svc_rdma_init(void)
 	kmem_cache_destroy(svc_rdma_map_cachep);
  err0:
 	unregister_sysctl_table(svcrdma_table_header);
+	destroy_workqueue(svc_rdma_wq);
 	return -ENOMEM;
 }
 MODULE_AUTHOR("Tom Tucker <[email protected]>");
net/sunrpc/xprtrdma/svc_rdma_transport.c +5 -1
Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================
@@ -45,6 +45,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/workqueue.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include <linux/sunrpc/svc_rdma.h>
@@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
 /* WR context cache. Created in svc_rdma.c  */
 extern struct kmem_cache *svc_rdma_ctxt_cachep;
 
+/* Workqueue created in svc_rdma.c */
+extern struct workqueue_struct *svc_rdma_wq;
+
 struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
 {
 	struct svc_rdma_op_ctxt *ctxt;
@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
 	struct svcxprt_rdma *rdma =
 		container_of(xprt, struct svcxprt_rdma, sc_xprt);
 	INIT_WORK(&rdma->sc_work, __svc_rdma_free);
-	schedule_work(&rdma->sc_work);
+	queue_work(svc_rdma_wq, &rdma->sc_work);
 }
 
 static int svc_rdma_has_wspace(struct svc_xprt *xprt)
net/sunrpc/xprtrdma/transport.c +2 -3
Index: work/net/sunrpc/xprtrdma/transport.c
===================================================================
@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
 	dprintk("RPC:       %s: called\n", __func__);
 
-	cancel_delayed_work(&r_xprt->rdma_connect);
-	flush_scheduled_work();
+	cancel_delayed_work_sync(&r_xprt->rdma_connect);
 
 	xprt_clear_connected(xprt);
 
@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
 	} else {
 		schedule_delayed_work(&r_xprt->rdma_connect, 0);
 		if (!RPC_IS_ASYNC(task))
-			flush_scheduled_work();
+			flush_delayed_work(&r_xprt->rdma_connect);
 	}
 }
 
#1 lore raw

Tom, could you take a look?

--b.

Quoted 130 lines · Tejun Heo

* Create and use svc_rdma_wq instead of using the system workqueue and flush_scheduled_work(). This workqueue is necessary to serve as flushing domain for rdma->sc_work which is used to destroy itself and thus can't be flushed explicitly.

* Replace cancel_delayed_work() + flush_scheduled_work() with cancel_delayed_work_sync().

* Implement synchronous connect in xprt_rdma_connect() using flush_delayed_work() on the rdma_connect work instead of using flush_scheduled_work().

This is to prepare for the deprecation and removal of flush_scheduled_work().

---
 net/sunrpc/xprtrdma/svc_rdma.c           |   11 ++++++++++-
 net/sunrpc/xprtrdma/svc_rdma_transport.c |    6 +++++-
 net/sunrpc/xprtrdma/transport.c          |    5 ++---
 3 files changed, 17 insertions(+), 5 deletions(-)

Index: work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================

net/sunrpc/xprtrdma/svc_rdma.c +10 -1
@@ -43,6 +43,7 @@
 #include <linux/slab.h>
 #include <linux/fs.h>
 #include <linux/sysctl.h>
+#include <linux/workqueue.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/sched.h>
 #include <linux/sunrpc/svc_rdma.h>
@@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
 struct kmem_cache *svc_rdma_map_cachep;
 struct kmem_cache *svc_rdma_ctxt_cachep;
 
+struct workqueue_struct *svc_rdma_wq;
+
 /*
  * This function implements reading and resetting an atomic_t stat
  * variable through read/write to a proc file. Any write to the file
@@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
 void svc_rdma_cleanup(void)
 {
 	dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
-	flush_scheduled_work();
+	destroy_workqueue(svc_rdma_wq);
 	if (svcrdma_table_header) {
 		unregister_sysctl_table(svcrdma_table_header);
 		svcrdma_table_header = NULL;
@@ -249,6 +252,11 @@ int svc_rdma_init(void)
 	dprintk("\tsq_depth         : %d\n",
 		svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
 	dprintk("\tmax_inline       : %d\n", svcrdma_max_req_size);
+
+	svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
+	if (!svc_rdma_wq)
+		return -ENOMEM;
+
 	if (!svcrdma_table_header)
 		svcrdma_table_header =
 			register_sysctl_table(svcrdma_root_table);
@@ -283,6 +291,7 @@ int svc_rdma_init(void)
 	kmem_cache_destroy(svc_rdma_map_cachep);
  err0:
 	unregister_sysctl_table(svcrdma_table_header);
+	destroy_workqueue(svc_rdma_wq);
 	return -ENOMEM;
 }
 MODULE_AUTHOR("Tom Tucker <[email protected]>");

Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================

net/sunrpc/xprtrdma/svc_rdma_transport.c +5 -1
@@ -45,6 +45,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/workqueue.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include <linux/sunrpc/svc_rdma.h>
@@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
 /* WR context cache. Created in svc_rdma.c  */
 extern struct kmem_cache *svc_rdma_ctxt_cachep;
 
+/* Workqueue created in svc_rdma.c */
+extern struct workqueue_struct *svc_rdma_wq;
+
 struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
 {
 	struct svc_rdma_op_ctxt *ctxt;
@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
 	struct svcxprt_rdma *rdma =
 		container_of(xprt, struct svcxprt_rdma, sc_xprt);
 	INIT_WORK(&rdma->sc_work, __svc_rdma_free);
-	schedule_work(&rdma->sc_work);
+	queue_work(svc_rdma_wq, &rdma->sc_work);
 }
 
 static int svc_rdma_has_wspace(struct svc_xprt *xprt)

Index: work/net/sunrpc/xprtrdma/transport.c
===================================================================

net/sunrpc/xprtrdma/transport.c +2 -4
@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
 	dprintk("RPC:       %s: called\n", __func__);
 
-	cancel_delayed_work(&r_xprt->rdma_connect);
-	flush_scheduled_work();
+	cancel_delayed_work_sync(&r_xprt->rdma_connect);
 
 	xprt_clear_connected(xprt);
 
@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
 	} else {
 		schedule_delayed_work(&r_xprt->rdma_connect, 0);
 		if (!RPC_IS_ASYNC(task))
-			flush_scheduled_work();
+			flush_delayed_work(&r_xprt->rdma_connect);
 	}
 }
 
--

To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#2 lore raw
Quoted 2 lines · J. Bruce Fields

Tom, could you take a look?

This looks reasonable to me.

My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.

Tom

Quoted 138 lines

--b.

Quoted 127 lines · Tejun Heo

* Create and use svc_rdma_wq instead of using the system workqueue and flush_scheduled_work(). This workqueue is necessary to serve as flushing domain for rdma->sc_work which is used to destroy itself and thus can't be flushed explicitly.

* Replace cancel_delayed_work() + flush_scheduled_work() with cancel_delayed_work_sync().

* Implement synchronous connect in xprt_rdma_connect() using flush_delayed_work() on the rdma_connect work instead of using flush_scheduled_work().

This is to prepare for the deprecation and removal of flush_scheduled_work().

---
  net/sunrpc/xprtrdma/svc_rdma.c           |   11 ++++++++++-
  net/sunrpc/xprtrdma/svc_rdma_transport.c |    6 +++++-
  net/sunrpc/xprtrdma/transport.c          |    5 ++---
  3 files changed, 17 insertions(+), 5 deletions(-)

Index: work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================

net/sunrpc/xprtrdma/svc_rdma.c +10 -1
@@ -43,6 +43,7 @@
  #include<linux/slab.h>
  #include<linux/fs.h>
  #include<linux/sysctl.h>
+#include<linux/workqueue.h>
  #include<linux/sunrpc/clnt.h>
  #include<linux/sunrpc/sched.h>
  #include<linux/sunrpc/svc_rdma.h>
@@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;
  struct kmem_cache *svc_rdma_map_cachep;
  struct kmem_cache *svc_rdma_ctxt_cachep;
 
+struct workqueue_struct *svc_rdma_wq;
+
  /*
   * This function implements reading and resetting an atomic_t stat
   * variable through read/write to a proc file. Any write to the file
@@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =
  void svc_rdma_cleanup(void)
  {
  	dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
-	flush_scheduled_work();
+	destroy_workqueue(svc_rdma_wq);
  	if (svcrdma_table_header) {
  		unregister_sysctl_table(svcrdma_table_header);
  		svcrdma_table_header = NULL;
@@ -249,6 +252,11 @@ int svc_rdma_init(void)
  	dprintk("\tsq_depth         : %d\n",
  		svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
  	dprintk("\tmax_inline       : %d\n", svcrdma_max_req_size);
+
+	svc_rdma_wq = alloc_workqueue("svc_rdma", 0, 0);
+	if (!svc_rdma_wq)
+		return -ENOMEM;
+
  	if (!svcrdma_table_header)
  		svcrdma_table_header =
  			register_sysctl_table(svcrdma_root_table);
@@ -283,6 +291,7 @@ int svc_rdma_init(void)
  	kmem_cache_destroy(svc_rdma_map_cachep);
   err0:
  	unregister_sysctl_table(svcrdma_table_header);
+	destroy_workqueue(svc_rdma_wq);
  	return -ENOMEM;
  }
  MODULE_AUTHOR("Tom Tucker<[email protected]>");

Index: work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================

net/sunrpc/xprtrdma/svc_rdma_transport.c +5 -1
@@ -45,6 +45,7 @@
  #include<linux/sched.h>
  #include<linux/slab.h>
  #include<linux/spinlock.h>
+#include<linux/workqueue.h>
  #include<rdma/ib_verbs.h>
  #include<rdma/rdma_cm.h>
  #include<linux/sunrpc/svc_rdma.h>
@@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {
  /* WR context cache. Created in svc_rdma.c  */
  extern struct kmem_cache *svc_rdma_ctxt_cachep;
 
+/* Workqueue created in svc_rdma.c */
+extern struct workqueue_struct *svc_rdma_wq;
+
  struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
  {
  	struct svc_rdma_op_ctxt *ctxt;
@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xpr
  	struct svcxprt_rdma *rdma =
  		container_of(xprt, struct svcxprt_rdma, sc_xprt);
  	INIT_WORK(&rdma->sc_work, __svc_rdma_free);
-	schedule_work(&rdma->sc_work);
+	queue_work(svc_rdma_wq,&rdma->sc_work);
  }
 
  static int svc_rdma_has_wspace(struct svc_xprt *xprt)

Index: work/net/sunrpc/xprtrdma/transport.c
===================================================================

net/sunrpc/xprtrdma/transport.c +2 -4
@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
  	dprintk("RPC:       %s: called\n", __func__);
 
-	cancel_delayed_work(&r_xprt->rdma_connect);
-	flush_scheduled_work();
+	cancel_delayed_work_sync(&r_xprt->rdma_connect);
 
  	xprt_clear_connected(xprt);
 
@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task)
  	} else {
  		schedule_delayed_work(&r_xprt->rdma_connect, 0);
  		if (!RPC_IS_ASYNC(task))
-			flush_scheduled_work();
+			flush_delayed_work(&r_xprt->rdma_connect);
  	}
  }
 
--
Quoted 3 lines

To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

author
#3 lore raw
Quoted 8 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields

Tom, could you take a look?

This looks reasonable to me.

My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.

Yeap, only compile tested. Testing would be much appreciated.

Thanks.

Signature · 1 line
tejun
#4 lore raw
Quoted 11 lines · Tejun Heo
Quoted 8 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields

Tom, could you take a look?

This looks reasonable to me.

My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.

Yeap, only compile tested. Testing would be much appreciated.

Tejun,

I've tested this now with iozone etc... works like a charm ;-)

Tom

Quoted 2 lines

Thanks.

#5 lore raw
Quoted 7 lines · Tom Tucker
Quoted 6 lines · Tejun Heo
Quoted 5 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields

Tom, could you take a look?

This looks reasonable to me.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.

Yeap, only compile tested. Testing would be much appreciated.

Tejun,

I've tested this now with iozone etc... works like a charm ;-)

Applied, thanks!

--b.

author
#6 lore raw
Quoted 3 lines · J. Bruce Fields
Quoted 1 line

I've tested this now with iozone etc... works like a charm ;-)

Applied, thanks!

Great, thank you. :-)

Signature · 1 line
tejun