sunrpc/xprtrdma: clean up workqueue usage
Subsystems (3)
PATCH 1/1 sunrpc/xprtrdma: clean up workqueue usage
* Create and use svc_rdma_wq instead of using the system workqueue and flush_scheduled_work(). This workqueue is necessary to serve as flushing domain for rdma->sc_work which is used to destroy itself and thus can't be flushed explicitly.
* Replace cancel_delayed_work() + flush_scheduled_work() with cancel_delayed_work_sync().
* Implement synchronous connect in xprt_rdma_connect() using flush_delayed_work() on the rdma_connect work instead of using flush_scheduled_work().
This is to prepare for the deprecation and removal of flush_scheduled_work().
--- net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++- net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++- net/sunrpc/xprtrdma/transport.c | 5 ++--- 3 files changed, 17 insertions(+), 5 deletions(-)
net/sunrpc/xprtrdma/svc_rdma.c +10 -1
@@ -43,6 +43,7 @@#include <linux/slab.h> #include <linux/fs.h> #include <linux/sysctl.h> +#include <linux/workqueue.h> #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/sched.h> #include <linux/sunrpc/svc_rdma.h>
@@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;struct kmem_cache *svc_rdma_map_cachep;struct kmem_cache *svc_rdma_ctxt_cachep; +struct workqueue_struct *svc_rdma_wq; +/* * This function implements reading and resetting an atomic_t stat * variable through read/write to a proc file. Any write to the file
@@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =void svc_rdma_cleanup (void ) {dprintk ("SVCRDMA Module Removed, deregister RPC RDMA transport\n" ); -flush_scheduled_work (); +destroy_workqueue (svc_rdma_wq);if (svcrdma_table_header) {unregister_sysctl_table (svcrdma_table_header); svcrdma_table_header =NULL ;
@@ -249,6 +252,11 @@ int svc_rdma_init(void)dprintk ("\tsq_depth : %d\n" , svcrdma_max_requests *RPCRDMA_SQ_DEPTH_MULT );dprintk ("\tmax_inline : %d\n" , svcrdma_max_req_size); + + svc_rdma_wq =alloc_workqueue ("svc_rdma" ,0 ,0 ); +if (!svc_rdma_wq) +return -ENOMEM ; +if (!svcrdma_table_header) svcrdma_table_header =register_sysctl_table (svcrdma_root_table);
@@ -283,6 +291,7 @@ int svc_rdma_init(void)kmem_cache_destroy (svc_rdma_map_cachep); err0:unregister_sysctl_table (svcrdma_table_header); +destroy_workqueue (svc_rdma_wq);return -ENOMEM ; }MODULE_AUTHOR ("Tom Tucker <[email protected]>" );
net/sunrpc/xprtrdma/svc_rdma_transport.c +5 -1
@@ -45,6 +45,7 @@#include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/workqueue.h> #include <rdma/ib_verbs.h> #include <rdma/rdma_cm.h> #include <linux/sunrpc/svc_rdma.h>
@@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {/* WR context cache. Created in svc_rdma.c */ extern struct kmem_cache *svc_rdma_ctxt_cachep; +/* Workqueue created in svc_rdma.c */ +extern struct workqueue_struct *svc_rdma_wq; +struct svc_rdma_op_ctxt *svc_rdma_get_context (struct svcxprt_rdma *xprt) {struct svc_rdma_op_ctxt *ctxt;
@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xprstruct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt);INIT_WORK (&rdma->sc_work, __svc_rdma_free); -schedule_work (&rdma->sc_work); +queue_work (svc_rdma_wq, &rdma->sc_work); }static int svc_rdma_has_wspace (struct svc_xprt *xprt)
net/sunrpc/xprtrdma/transport.c +2 -3
@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)dprintk ("RPC: %s: called\n" , __func__); -cancel_delayed_work (&r_xprt->rdma_connect); -flush_scheduled_work (); +cancel_delayed_work_sync (&r_xprt->rdma_connect);xprt_clear_connected (xprt);
@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task) }else {schedule_delayed_work (&r_xprt->rdma_connect,0 );if (!RPC_IS_ASYNC (task)) -flush_scheduled_work (); +flush_delayed_work (&r_xprt->rdma_connect); } }
Tom, could you take a look?
--b.
Quoted 130 lines · Tejun Heo
* Create and use
svc_rdma_wqinstead of using the system workqueue andflush_scheduled_work(). This workqueue is necessary to serve as flushing domain forrdma->sc_workwhich is used to destroy itself and thus can't be flushed explicitly.* Replace
cancel_delayed_work()+flush_scheduled_work()withcancel_delayed_work_sync().* Implement synchronous connect in
xprt_rdma_connect()usingflush_delayed_work()on therdma_connectwork instead of usingflush_scheduled_work().This is to prepare for the deprecation and removal of
flush_scheduled_work().Signed-off-by Tejun Heo <[email protected]>--- net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++- net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++- net/sunrpc/xprtrdma/transport.c | 5 ++--- 3 files changed, 17 insertions(+), 5 deletions(-)Index:
work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================net/sunrpc/xprtrdma/svc_rdma.c +10
-1@@ -43,6 +43,7 @@#include <linux/slab.h> #include <linux/fs.h> #include <linux/sysctl.h> +#include <linux/workqueue.h> #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/sched.h> #include <linux/sunrpc/svc_rdma.h> @@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;struct kmem_cache *svc_rdma_map_cachep;struct kmem_cache *svc_rdma_ctxt_cachep; +struct workqueue_struct *svc_rdma_wq; +/* * This function implements reading and resetting an atomic_t stat * variable through read/write to a proc file. Any write to the file @@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =void svc_rdma_cleanup (void ) {dprintk ("SVCRDMA Module Removed, deregister RPC RDMA transport\n" ); -flush_scheduled_work (); +destroy_workqueue (svc_rdma_wq);if (svcrdma_table_header) {unregister_sysctl_table (svcrdma_table_header); svcrdma_table_header =NULL ;@@ -249,6 +252,11 @@ int svc_rdma_init(void)dprintk ("\tsq_depth : %d\n" , svcrdma_max_requests *RPCRDMA_SQ_DEPTH_MULT );dprintk ("\tmax_inline : %d\n" , svcrdma_max_req_size); + + svc_rdma_wq =alloc_workqueue ("svc_rdma" ,0 ,0 ); +if (!svc_rdma_wq) +return -ENOMEM ; +if (!svcrdma_table_header) svcrdma_table_header =register_sysctl_table (svcrdma_root_table);@@ -283,6 +291,7 @@ int svc_rdma_init(void)kmem_cache_destroy (svc_rdma_map_cachep); err0:unregister_sysctl_table (svcrdma_table_header); +destroy_workqueue (svc_rdma_wq);return -ENOMEM ; }MODULE_AUTHOR ("Tom Tucker <[email protected]>" );Index:
work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================net/sunrpc/xprtrdma/svc_rdma_transport.c +5
-1@@ -45,6 +45,7 @@#include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/workqueue.h> #include <rdma/ib_verbs.h> #include <rdma/rdma_cm.h> #include <linux/sunrpc/svc_rdma.h> @@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {/* WR context cache. Created in svc_rdma.c */ extern struct kmem_cache *svc_rdma_ctxt_cachep; +/* Workqueue created in svc_rdma.c */ +extern struct workqueue_struct *svc_rdma_wq; +struct svc_rdma_op_ctxt *svc_rdma_get_context (struct svcxprt_rdma *xprt) {struct svc_rdma_op_ctxt *ctxt;@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xprstruct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt);INIT_WORK (&rdma->sc_work, __svc_rdma_free); -schedule_work (&rdma->sc_work); +queue_work (svc_rdma_wq, &rdma->sc_work); }static int svc_rdma_has_wspace (struct svc_xprt *xprt)Index:
work/net/sunrpc/xprtrdma/transport.c
===================================================================net/sunrpc/xprtrdma/transport.c +2
-4@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)dprintk ("RPC: %s: called\n" , __func__); -cancel_delayed_work (&r_xprt->rdma_connect); -flush_scheduled_work (); +cancel_delayed_work_sync (&r_xprt->rdma_connect);xprt_clear_connected (xprt);@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task) }else {schedule_delayed_work (&r_xprt->rdma_connect,0 );if (!RPC_IS_ASYNC (task)) -flush_scheduled_work (); +flush_delayed_work (&r_xprt->rdma_connect); } } --To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 2 lines · J. Bruce Fields
Tom, could you take a look?
This looks reasonable to me.
My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.
Tom
Quoted 138 lines
--b.
Quoted 127 lines · Tejun Heo
* Create and use
svc_rdma_wqinstead of using the system workqueue andflush_scheduled_work(). This workqueue is necessary to serve as flushing domain forrdma->sc_workwhich is used to destroy itself and thus can't be flushed explicitly.* Replace
cancel_delayed_work()+flush_scheduled_work()withcancel_delayed_work_sync().* Implement synchronous connect in
xprt_rdma_connect()usingflush_delayed_work()on therdma_connectwork instead of usingflush_scheduled_work().This is to prepare for the deprecation and removal of
flush_scheduled_work().Signed-off-by Tejun Heo<[email protected]>--- net/sunrpc/xprtrdma/svc_rdma.c | 11 ++++++++++- net/sunrpc/xprtrdma/svc_rdma_transport.c | 6 +++++- net/sunrpc/xprtrdma/transport.c | 5 ++--- 3 files changed, 17 insertions(+), 5 deletions(-)Index:
work/net/sunrpc/xprtrdma/svc_rdma.c
===================================================================net/sunrpc/xprtrdma/svc_rdma.c +10
-1@@ -43,6 +43,7 @@#include <linux/slab.h> #include <linux/fs.h> #include <linux/sysctl.h> +#include <linux/workqueue.h> #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/sched.h> #include <linux/sunrpc/svc_rdma.h> @@ -74,6 +75,8 @@ atomic_t rdma_stat_sq_prod;struct kmem_cache *svc_rdma_map_cachep;struct kmem_cache *svc_rdma_ctxt_cachep; +struct workqueue_struct *svc_rdma_wq; +/* * This function implements reading and resetting an atomic_t stat * variable through read/write to a proc file. Any write to the file @@ -231,7 +234,7 @@ static ctl_table svcrdma_root_table[] =void svc_rdma_cleanup (void ) {dprintk ("SVCRDMA Module Removed, deregister RPC RDMA transport\n" ); -flush_scheduled_work (); +destroy_workqueue (svc_rdma_wq);if (svcrdma_table_header) {unregister_sysctl_table (svcrdma_table_header); svcrdma_table_header =NULL ;@@ -249,6 +252,11 @@ int svc_rdma_init(void)dprintk ("\tsq_depth : %d\n" , svcrdma_max_requests *RPCRDMA_SQ_DEPTH_MULT );dprintk ("\tmax_inline : %d\n" , svcrdma_max_req_size); + + svc_rdma_wq =alloc_workqueue ("svc_rdma" ,0 ,0 ); +if (!svc_rdma_wq) +return -ENOMEM ; +if (!svcrdma_table_header) svcrdma_table_header =register_sysctl_table (svcrdma_root_table);@@ -283,6 +291,7 @@ int svc_rdma_init(void)kmem_cache_destroy (svc_rdma_map_cachep); err0:unregister_sysctl_table (svcrdma_table_header); +destroy_workqueue (svc_rdma_wq);return -ENOMEM ; }MODULE_AUTHOR ("Tom Tucker<[email protected]>" );Index:
work/net/sunrpc/xprtrdma/svc_rdma_transport.c
===================================================================net/sunrpc/xprtrdma/svc_rdma_transport.c +5
-1@@ -45,6 +45,7 @@#include <linux/sched.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/workqueue.h> #include <rdma/ib_verbs.h> #include <rdma/rdma_cm.h> #include <linux/sunrpc/svc_rdma.h> @@ -89,6 +90,9 @@ struct svc_xprt_class svc_rdma_class = {/* WR context cache. Created in svc_rdma.c */ extern struct kmem_cache *svc_rdma_ctxt_cachep; +/* Workqueue created in svc_rdma.c */ +extern struct workqueue_struct *svc_rdma_wq; +struct svc_rdma_op_ctxt *svc_rdma_get_context (struct svcxprt_rdma *xprt) {struct svc_rdma_op_ctxt *ctxt;@@ -1184,7 +1188,7 @@ static void svc_rdma_free(struct svc_xprstruct svcxprt_rdma *rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt);INIT_WORK (&rdma->sc_work, __svc_rdma_free); -schedule_work (&rdma->sc_work); +queue_work (svc_rdma_wq,&rdma->sc_work); }static int svc_rdma_has_wspace (struct svc_xprt *xprt)Index:
work/net/sunrpc/xprtrdma/transport.c
===================================================================net/sunrpc/xprtrdma/transport.c +2
-4@@ -237,8 +237,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)dprintk ("RPC: %s: called\n" , __func__); -cancel_delayed_work (&r_xprt->rdma_connect); -flush_scheduled_work (); +cancel_delayed_work_sync (&r_xprt->rdma_connect);xprt_clear_connected (xprt);@@ -460,7 +459,7 @@ xprt_rdma_connect(struct rpc_task *task) }else {schedule_delayed_work (&r_xprt->rdma_connect,0 );if (!RPC_IS_ASYNC (task)) -flush_scheduled_work (); +flush_delayed_work (&r_xprt->rdma_connect); } } --Quoted 3 lines
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Quoted 8 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields
Tom, could you take a look?
This looks reasonable to me.
My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.
Yeap, only compile tested. Testing would be much appreciated.
Thanks.
Signature · 1 line
tejun
Quoted 11 lines · Tejun Heo
Quoted 8 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields
Tom, could you take a look?
This looks reasonable to me.
My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.
Yeap, only compile tested. Testing would be much appreciated.
Tejun,
I've tested this now with iozone etc... works like a charm ;-)
Tom
Quoted 2 lines
Thanks.
J. Bruce Fields
AppliedQuoted 7 lines · Tom Tucker
Quoted 6 lines · Tejun Heo
Quoted 5 lines · Tom Tucker
Quoted 2 lines · J. Bruce Fields
Tom, could you take a look?
This looks reasonable to me.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
My bet is that it wasn't tested though due to hardware availability. I'll test it this weekend and let you know how it goes.
Yeap, only compile tested. Testing would be much appreciated.
Tejun,
I've tested this now with iozone etc... works like a charm ;-)
Applied, thanks!
--b.
Quoted 3 lines · J. Bruce Fields
Quoted 1 line
I've tested this now with iozone etc... works like a charm ;-)
Applied, thanks!
Great, thank you. :-)
Signature · 1 line
tejun