workqueue: don't always set __WQ_ORDERED implicitly
PATCH 1/2 workqueue: don't always set __WQ_ORDERED implicitly
Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.
E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.
This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.
--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +4 -0
@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name) +#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );
kernel/workqueue.c +3 -1
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient)
Signature · 1 line
2.9.5
PATCH 2/2 scsi: register sysfs for scsi/iscsi workqueues
This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
- iscsi_q_xx
- iscsi_eh
--- drivers/scsi/hosts.c | 4 ++-- drivers/scsi/libiscsi.c | 2 +- drivers/scsi/scsi_transport_iscsi.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
drivers/scsi/hosts.c +2 -2
@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,if (shost->transportt->create_work_queue) {snprintf (shost->work_q_name,sizeof (shost->work_q_name),"scsi_wq_%d" , shost->host_no); - shost->work_q =create_singlethread_workqueue ( + shost->work_q =create_singlethread_workqueue_noorder ( shost->work_q_name);if (!shost->work_q) { error = -EINVAL ;
@@ -487,7 +487,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize) } shost->tmf_work_q =alloc_workqueue ("scsi_tmf_%d" , -WQ_UNBOUND |WQ_MEM_RECLAIM , +WQ_UNBOUND |WQ_MEM_RECLAIM |WQ_SYSFS | __WQ_ORDERED_DISABLE,1 , shost->host_no);if (!shost->tmf_work_q) { shost_printk(KERN_WARNING, shost,
drivers/scsi/libiscsi.c +1 -1
@@ -2627,7 +2627,7 @@ struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,if (xmit_can_sleep) {snprintf (ihost->workq_name,sizeof (ihost->workq_name),"iscsi_q_%d" , shost->host_no); - ihost->workq =create_singlethread_workqueue (ihost->workq_name); + ihost->workq =create_singlethread_workqueue_noorder (ihost->workq_name);if (!ihost->workq) goto free_host; }
drivers/scsi/scsi_transport_iscsi.c +1 -1
@@ -4602,7 +4602,7 @@ static __init int iscsi_transport_init(void) goto unregister_flashnode_bus; } - iscsi_eh_timer_workq =create_singlethread_workqueue ("iscsi_eh" ); + iscsi_eh_timer_workq =create_singlethread_workqueue_noorder ("iscsi_eh" );if (!iscsi_eh_timer_workq) { err = -ENOMEM ; goto release_nls;
Signature · 1 line
2.9.5
Mike Christie
maintainerRe: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues
Quoted 19 lines · Bob Liu
This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
-iscsi_q_xx
-iscsi_ehSigned-off-by Bob Liu <[email protected]>--- drivers/scsi/hosts.c | 4 ++-- drivers/scsi/libiscsi.c | 2 +- drivers/scsi/scsi_transport_iscsi.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)drivers/scsi/hosts.c +0
-0
drivers/scsi/hosts.c int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, · line 278
Quoted 8 lines
drivers/scsi/hosts.c +1
-1@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,if (shost->transportt->create_work_queue) {snprintf (shost->work_q_name,sizeof (shost->work_q_name),"scsi_wq_%d" , shost->host_no); - shost->work_q =create_singlethread_workqueue ( + shost->work_q =create_singlethread_workqueue_noorder ( shost->work_q_name);if (!shost->work_q) { error = -EINVAL ;
This patch seems ok for the iscsi, fc, tmf, and non transport class scan uses. We are either heavy handed with flushes or did not need ordering.
I don't know about the zfcp use though, so I cc'd the developers listed as maintainers. It looks like for zfcp we can do:
zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_work to scan the scsi target on the rport.
and then zfcp_scsi_rport_register can call zfcp_unit_queue_scsi_scan-> scsi_queue_work which will scan for a specific lun.
It looks ok if those are not ordered, but I would get their review to make sure.
Re: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues
Quoted 32 lines · Mike Christie
Quoted 19 lines · Bob Liu
This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
-iscsi_q_xx
-iscsi_ehSigned-off-by Bob Liu <[email protected]>--- drivers/scsi/hosts.c | 4 ++-- drivers/scsi/libiscsi.c | 2 +- drivers/scsi/scsi_transport_iscsi.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)drivers/scsi/hosts.c +0
-0drivers/scsi/hosts.c int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, · line 278
Quoted 8 lines
drivers/scsi/hosts.c +1
-1@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,if (shost->transportt->create_work_queue) {snprintf (shost->work_q_name,sizeof (shost->work_q_name),"scsi_wq_%d" , shost->host_no); - shost->work_q =create_singlethread_workqueue ( + shost->work_q =create_singlethread_workqueue_noorder ( shost->work_q_name);if (!shost->work_q) { error = -EINVAL ;This patch seems ok for the iscsi, fc, tmf, and non transport class scan uses. We are either heavy handed with flushes or did not need ordering.
Quoted 2 lines
I don't know about the zfcp use though, so I cc'd the developers listed as maintainers. It looks like for zfcp we can do:
Thx for the notice.
Quoted 9 lines
zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_workto scan the scsi target on the rport.and then
zfcp_scsi_rport_registercan call zfcp_unit_queue_scsi_scan->scsi_queue_workwhich will scan for a specific lun.It looks ok if those are not ordered, but I would get their review to make sure.
I am not aware of any temporal requirements of those LUN-scans, so I think making them not explicitly ordered shouldn't hurt us.
The target scan itself is protected again by shost->scan_mutex.. so all fine I think.
Signature · 4 lines
Best Regards, Benjamin Block / Linux on IBM Z Kernel Development / IBM Systems IBM Deutschland Research & Development GmbH / https://www.ibm.com/privacy Vorsitz. AufsR.: Gregor Pillen / Geschäftsführung: Dirk Wittkopp Sitz der Gesellschaft: Böblingen / Registergericht: AmtsG Stuttgart, HRB 243294
ping..
Quoted 54 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +4
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name) +#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );kernel/workqueue.c +3
-1@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient)
Quoted 7 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.
Hello
If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.
Quoted 23 lines
This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +1
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */
include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439
Quoted 6 lines
include/linux/workqueue.h +3
-0@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name) +#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.
Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can be wrapped as a new function or macro, but I don't think create_singlethread_workqueue_noorder() is a good name for it.
Quoted 21 lines
include/linux/workqueue.h +0
-0extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );kernel/workqueue.c +3
-2@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient) --2.9.5
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.Hello
If I read the code correctly, the reason why their cpumask can't be changed is because
__WQ_ORDERED_EXPLICIT, not__WQ_ORDERED.Quoted 26 lines
This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +1
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
Quoted 7 lines
include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439
Quoted 3 lines
include/linux/workqueue.h +3
-0+#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))I think using
__WQ_ORDEREDwithout__WQ_ORDERED_EXPLICITis what you need, in which case cpumask is allowed to be changed.
I don't think so, see function workqueue_apply_unbound_cpumask():
wq_unbound_cpumask_store()
Quoted 2 lines
workqueue_set_unbound_cpumask()workqueue_apply_unbound_cpumask(){
...
5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags & __WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only __WQ_ORDERED is set.
5280 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
}
Thanks for your review.
Bob
Quoted 24 lines
Just use
alloc_workqueue()with__WQ_ORDEREDand max_active=1. It can
be wrapped as a new function or macro, but I don't think>create_singlethread_workqueue_noorder()is a good name for it.Quoted 21 lines
include/linux/workqueue.h +0
-0extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );kernel/workqueue.c +3
-2@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient) --2.9.5
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.Hello
If I read the code correctly, the reason why their cpumask can't be changed is because
__WQ_ORDERED_EXPLICIT, not__WQ_ORDERED.Quoted 26 lines
This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +1
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)Quoted 7 lines
include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439
Quoted 3 lines
include/linux/workqueue.h +3
-0+#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))I think using
__WQ_ORDEREDwithout__WQ_ORDERED_EXPLICITis what you need, in which case cpumask is allowed to be changed.I don't think so, see function
workqueue_apply_unbound_cpumask():
wq_unbound_cpumask_store()
Quoted 8 lines
Quoted 2 lines
workqueue_set_unbound_cpumask()workqueue_apply_unbound_cpumask(){...5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags&__WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only__WQ_ORDEREDis set.
wq_unbound_cpumask_store() is for changing the cpumask of *all* workqueues. I don't think it can be used to make scsi and iscsi workqueues bound to different cpu.
apply_workqueue_attrs() is for changing the cpumask of the specific workqueue, which can change the cpumask of __WQ_ORDERED workqueue (but without __WQ_ORDERED_EXPLICIT).
Quoted 33 lines
5280 ctx =
apply_wqattrs_prepare(wq, wq->unbound_attrs);}Thanks for your review.
BobQuoted 24 lines
Just use
alloc_workqueue()with__WQ_ORDEREDand max_active=1. It can
be wrapped as a new function or macro, but I don't think>create_singlethread_workqueue_noorder()is a good name for it.Quoted 21 lines
include/linux/workqueue.h +0
-0extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );kernel/workqueue.c +3
-2@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient) --2.9.5
Quoted 63 lines · Lai Jiangshan
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.Hello
If I read the code correctly, the reason why their cpumask can't be changed is because
__WQ_ORDERED_EXPLICIT, not__WQ_ORDERED.Quoted 26 lines
This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +1
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)Quoted 7 lines
include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439
Quoted 3 lines
include/linux/workqueue.h +3
-0+#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))I think using
__WQ_ORDEREDwithout__WQ_ORDERED_EXPLICITis what you need, in which case cpumask is allowed to be changed.I don't think so, see function
workqueue_apply_unbound_cpumask():
wq_unbound_cpumask_store()Quoted 8 lines
Quoted 2 lines
workqueue_set_unbound_cpumask()workqueue_apply_unbound_cpumask(){...5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags&__WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only__WQ_ORDEREDis set.
Quoted 2 lines
wq_unbound_cpumask_store()is for changing the cpumask of *all* workqueues.
Isn't '/sys/bus/workqueue/devices/xxxx/cpumask' using the same function to change cpumask of specific workqueue?
Am I missing something..
Quoted 3 lines
I don't think it can be used to make
scsi and iscsi workqueues bound to different cpu.
The idea is to register scsi/iscsi workqueues with WQ_SYSFS, and then they can be bounded to different cpu by writing cpu number to "/sys/bus/workqueue/devices/xxxx/cpumask".
Quoted 37 lines
apply_workqueue_attrs()is for changing the cpumask of the specific workqueue, which can change the cpumask of__WQ_ORDEREDworkqueue (but without__WQ_ORDERED_EXPLICIT).Quoted 33 lines
5280 ctx =
apply_wqattrs_prepare(wq, wq->unbound_attrs);}Thanks for your review.
BobQuoted 24 lines
Just use
alloc_workqueue()with__WQ_ORDEREDand max_active=1. It can
be wrapped as a new function or macro, but I don't think>create_singlethread_workqueue_noorder()is a good name for it.Quoted 21 lines
include/linux/workqueue.h +0
-0extern void destroy_workqueue (struct workqueue_struct *wq);struct workqueue_attrs *alloc_workqueue_attrs (void );kernel/workqueue.c +3
-2@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,* on NUMA. */ if ((flags &WQ_UNBOUND ) && max_active ==1 ) - flags |= __WQ_ORDERED; +/* the caller may don't want __WQ_ORDERED to be set implicitly. */ +if (!(flags & __WQ_ORDERED_DISABLE)) + flags |= __WQ_ORDERED;/* see the comment above the definition of WQ_POWER_EFFICIENT */ if ((flags &WQ_POWER_EFFICIENT ) && wq_power_efficient) --2.9.5
Quoted 63 lines · Lai Jiangshan
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu
Current code always set 'Unbound &&
max_active== 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.E.g some scsi and iscsi
workqueues(unbound && max_active = 1)want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed becauseWQ_ORDEREDis set implicitly.Hello
If I read the code correctly, the reason why their cpumask can't be changed is because
__WQ_ORDERED_EXPLICIT, not__WQ_ORDERED.Quoted 26 lines
This patch adds a flag
__WQ_ORDERED_DISABLEand alsocreate_singlethread_workqueue_noorder()to offer an new option.Signed-off-by Bob Liu <[email protected]>--- include/linux/workqueue.h | 4 ++++ kernel/workqueue.c | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-)include/linux/workqueue.h +1
-0@@ -344,6 +344,7 @@ enum { __WQ_ORDERED =1 <<17 ,/* internal: workqueue is ordered */ __WQ_LEGACY =1 <<18 ,/* internal: create*_workqueue() */ __WQ_ORDERED_EXPLICIT =1 <<19 ,/* internal: alloc_ordered_workqueue() */ + __WQ_ORDERED_DISABLE =1 <<20 ,/* internal: don't set __WQ_ORDERED implicitly */ WQ_MAX_ACTIVE =512 ,/* I like 512, better ideas? */ WQ_MAX_UNBOUND_PER_CPU = 4, /* 4 * #cpus for unbound wq */@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,#define create_singlethread_workqueue (name) \ alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)Quoted 7 lines
include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439
Quoted 3 lines
include/linux/workqueue.h +3
-0+#define create_singlethread_workqueue_noorder (name) \ + alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \ + WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))I think using
__WQ_ORDEREDwithout__WQ_ORDERED_EXPLICITis what you need, in which case cpumask is allowed to be changed.I don't think so, see function
workqueue_apply_unbound_cpumask():
wq_unbound_cpumask_store()Quoted 8 lines
Quoted 2 lines
workqueue_set_unbound_cpumask()workqueue_apply_unbound_cpumask(){...5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags&__WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only__WQ_ORDEREDis set.
Quoted 8 lines
wq_unbound_cpumask_store()is for changing the cpumask of *all* workqueues. I don't think it can be used to make scsi and iscsi workqueues bound to different cpu.
apply_workqueue_attrs()is for changing the cpumask of the specific workqueue, which can change the cpumask of__WQ_ORDEREDworkqueue (but without__WQ_ORDERED_EXPLICIT).
Yes, you are right. I made a mistake.
Sorry for the noise.
Regards,
Bob
Quoted 4 lines
Quoted 4 lines
5280 ctx =
apply_wqattrs_prepare(wq, wq->unbound_attrs);}