Skip to contentSkip to conversation

workqueue: don't always set __WQ_ORDERED implicitly

PATCH In review Bob Liu 4 participants 10 messages Jun 11, 2020 – Jul 1, 2020

Subsystems (3)
lore
Share & tools
lore

Bob Liu

author
#0 lore raw

PATCH 1/2 workqueue: don't always set __WQ_ORDERED implicitly

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +4 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
 	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
 	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
 	__WQ_ORDERED_EXPLICIT	= 1 << 19, /* internal: alloc_ordered_workqueue() */
+	__WQ_ORDERED_DISABLE	= 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
 	WQ_MAX_ACTIVE		= 512,	  /* I like 512, better ideas? */
 	WQ_MAX_UNBOUND_PER_CPU	= 4,	  /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)				\
 	alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
+#define create_singlethread_workqueue_noorder(name)			\
+	alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+			WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -1
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 	 * on NUMA.
 	 */
 	if ((flags & WQ_UNBOUND) && max_active == 1)
-		flags |= __WQ_ORDERED;
+		/* the caller may don't want __WQ_ORDERED to be set implicitly. */
+		if (!(flags & __WQ_ORDERED_DISABLE))
+			flags |= __WQ_ORDERED;
 
 	/* see the comment above the definition of WQ_POWER_EFFICIENT */
 	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
Signature · 1 line
2.9.5
author
#1 lore raw

PATCH 2/2 scsi: register sysfs for scsi/iscsi workqueues

This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
- iscsi_q_xx
- iscsi_eh

---
 drivers/scsi/hosts.c                | 4 ++--
 drivers/scsi/libiscsi.c             | 2 +-
 drivers/scsi/scsi_transport_iscsi.c | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
drivers/scsi/hosts.c +2 -2
index 1d669e4..4b9f80d 100644
@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 	if (shost->transportt->create_work_queue) {
 		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
 			 "scsi_wq_%d", shost->host_no);
-		shost->work_q = create_singlethread_workqueue(
+		shost->work_q = create_singlethread_workqueue_noorder(
 					shost->work_q_name);
 		if (!shost->work_q) {
 			error = -EINVAL;
@@ -487,7 +487,7 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize)
 	}
 
 	shost->tmf_work_q = alloc_workqueue("scsi_tmf_%d",
-					    WQ_UNBOUND | WQ_MEM_RECLAIM,
+				WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS | __WQ_ORDERED_DISABLE,
 					   1, shost->host_no);
 	if (!shost->tmf_work_q) {
 		shost_printk(KERN_WARNING, shost,
drivers/scsi/libiscsi.c +1 -1
index 70b99c0..6808cf3 100644
@@ -2627,7 +2627,7 @@ struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,
 	if (xmit_can_sleep) {
 		snprintf(ihost->workq_name, sizeof(ihost->workq_name),
 			"iscsi_q_%d", shost->host_no);
-		ihost->workq = create_singlethread_workqueue(ihost->workq_name);
+		ihost->workq = create_singlethread_workqueue_noorder(ihost->workq_name);
 		if (!ihost->workq)
 			goto free_host;
 	}
drivers/scsi/scsi_transport_iscsi.c +1 -1
index dfc726f..d07a0e4 100644
@@ -4602,7 +4602,7 @@ static __init int iscsi_transport_init(void)
 		goto unregister_flashnode_bus;
 	}
 
-	iscsi_eh_timer_workq = create_singlethread_workqueue("iscsi_eh");
+	iscsi_eh_timer_workq = create_singlethread_workqueue_noorder("iscsi_eh");
 	if (!iscsi_eh_timer_workq) {
 		err = -ENOMEM;
 		goto release_nls;
Signature · 1 line
2.9.5
maintainer
#2 lore raw

Re: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues

Quoted 19 lines · Bob Liu

This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
- iscsi_q_xx
- iscsi_eh

---
  drivers/scsi/hosts.c                | 4 ++--
  drivers/scsi/libiscsi.c             | 2 +-
  drivers/scsi/scsi_transport_iscsi.c | 2 +-
  3 files changed, 4 insertions(+), 4 deletions(-)
drivers/scsi/hosts.c +0 -0
index 1d669e4..4b9f80d 100644

drivers/scsi/hosts.c int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, · line 278

Quoted 8 lines
drivers/scsi/hosts.c +1 -1
@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
  	if (shost->transportt->create_work_queue) {
  		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
  			 "scsi_wq_%d", shost->host_no);
-		shost->work_q = create_singlethread_workqueue(
+		shost->work_q = create_singlethread_workqueue_noorder(
  					shost->work_q_name);
  		if (!shost->work_q) {
  			error = -EINVAL;

This patch seems ok for the iscsi, fc, tmf, and non transport class scan uses. We are either heavy handed with flushes or did not need ordering.

I don't know about the zfcp use though, so I cc'd the developers listed as maintainers. It looks like for zfcp we can do:

zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_work to scan the scsi target on the rport.

and then zfcp_scsi_rport_register can call zfcp_unit_queue_scsi_scan-> scsi_queue_work which will scan for a specific lun.

It looks ok if those are not ordered, but I would get their review to make sure.

#3 lore raw

Re: [PATCH 2/2] scsi: register sysfs for scsi/iscsi workqueues

Quoted 32 lines · Mike Christie
Quoted 19 lines · Bob Liu

This patch enable setting cpu affinity through "cpumask" for below scsi/iscsi workqueues, so as to get better isolation.
- scsi_wq_*
- scsi_tmf_*
- iscsi_q_xx
- iscsi_eh

---
  drivers/scsi/hosts.c                | 4 ++--
  drivers/scsi/libiscsi.c             | 2 +-
  drivers/scsi/scsi_transport_iscsi.c | 2 +-
  3 files changed, 4 insertions(+), 4 deletions(-)
drivers/scsi/hosts.c +0 -0
index 1d669e4..4b9f80d 100644

drivers/scsi/hosts.c int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, · line 278

Quoted 8 lines
drivers/scsi/hosts.c +1 -1
@@ -272,7 +272,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
  	if (shost->transportt->create_work_queue) {
  		snprintf(shost->work_q_name, sizeof(shost->work_q_name),
  			 "scsi_wq_%d", shost->host_no);
-		shost->work_q = create_singlethread_workqueue(
+		shost->work_q = create_singlethread_workqueue_noorder(
  					shost->work_q_name);
  		if (!shost->work_q) {
  			error = -EINVAL;

This patch seems ok for the iscsi, fc, tmf, and non transport class scan uses. We are either heavy handed with flushes or did not need ordering.

Quoted 2 lines

I don't know about the zfcp use though, so I cc'd the developers listed as maintainers. It looks like for zfcp we can do:

Thx for the notice.

Quoted 9 lines

zfcp_scsi_rport_register->fc_remote_port_add->fc_remote_port_create->scsi_queue_work to scan the scsi target on the rport.

and then zfcp_scsi_rport_register can call zfcp_unit_queue_scsi_scan-> scsi_queue_work which will scan for a specific lun.

It looks ok if those are not ordered, but I would get their review to make sure.

I am not aware of any temporal requirements of those LUN-scans, so I think making them not explicitly ordered shouldn't hurt us.

The target scan itself is protected again by shost->scan_mutex.. so all fine I think.

Signature · 4 lines
Best Regards, Benjamin Block  / Linux on IBM Z Kernel Development / IBM Systems
IBM Deutschland Research & Development GmbH    /    https://www.ibm.com/privacy
Vorsitz. AufsR.: Gregor Pillen         /        Geschäftsführung: Dirk Wittkopp
Sitz der Gesellschaft: Böblingen / Registergericht: AmtsG Stuttgart, HRB 243294
author
#4 lore raw
In reply to Bob Liu

ping..

Quoted 54 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +4 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
 	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
 	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
 	__WQ_ORDERED_EXPLICIT	= 1 << 19, /* internal: alloc_ordered_workqueue() */
+	__WQ_ORDERED_DISABLE	= 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
 	WQ_MAX_ACTIVE		= 512,	  /* I like 512, better ideas? */
 	WQ_MAX_UNBOUND_PER_CPU	= 4,	  /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)				\
 	alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
+#define create_singlethread_workqueue_noorder(name)			\
+	alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+			WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -1
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 	 * on NUMA.
 	 */
 	if ((flags & WQ_UNBOUND) && max_active == 1)
-		flags |= __WQ_ORDERED;
+		/* the caller may don't want __WQ_ORDERED to be set implicitly. */
+		if (!(flags & __WQ_ORDERED_DISABLE))
+			flags |= __WQ_ORDERED;
 
 	/* see the comment above the definition of WQ_POWER_EFFICIENT */
 	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
#5 lore raw
In reply to Bob Liu
Quoted 7 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

Quoted 23 lines

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +1 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
        __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
        __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
        __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
+       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
        WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
        WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */

include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439

Quoted 6 lines
include/linux/workqueue.h +3 -0
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)                            \
        alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
+#define create_singlethread_workqueue_noorder(name)                    \
+       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.

Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can be wrapped as a new function or macro, but I don't think create_singlethread_workqueue_noorder() is a good name for it.

Quoted 21 lines
include/linux/workqueue.h +0 -0
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -2
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
         * on NUMA.
         */
        if ((flags & WQ_UNBOUND) && max_active == 1)
-               flags |= __WQ_ORDERED;
+               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
+               if (!(flags & __WQ_ORDERED_DISABLE))
+                       flags |= __WQ_ORDERED;
 
        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
--

2.9.5

author
#6 lore raw
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

Quoted 26 lines

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +1 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
        __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
        __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
        __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
+       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
        WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
        WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)                            \
        alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
Quoted 7 lines

include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439

Quoted 3 lines
include/linux/workqueue.h +3 -0
+#define create_singlethread_workqueue_noorder(name)                    \
+       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.

I don't think so, see function workqueue_apply_unbound_cpumask():

wq_unbound_cpumask_store()

Quoted 2 lines

workqueue_set_unbound_cpumask()
workqueue_apply_unbound_cpumask() {

     ...

5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags & __WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only __WQ_ORDERED is set.

5280 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);

     }

Thanks for your review.
Bob

Quoted 24 lines

Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.

Quoted 21 lines
include/linux/workqueue.h +0 -0
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -2
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
         * on NUMA.
         */
        if ((flags & WQ_UNBOUND) && max_active == 1)
-               flags |= __WQ_ORDERED;
+               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
+               if (!(flags & __WQ_ORDERED_DISABLE))
+                       flags |= __WQ_ORDERED;
 
        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
--

2.9.5

#7 lore raw
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

Quoted 26 lines

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +1 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
        __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
        __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
        __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
+       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
        WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
        WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)                            \
        alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
Quoted 7 lines

include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439

Quoted 3 lines
include/linux/workqueue.h +3 -0
+#define create_singlethread_workqueue_noorder(name)                    \
+       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.

I don't think so, see function workqueue_apply_unbound_cpumask():

wq_unbound_cpumask_store()

Quoted 8 lines
Quoted 2 lines

workqueue_set_unbound_cpumask()
workqueue_apply_unbound_cpumask() {

     ...

5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags & __WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only __WQ_ORDERED is set.

wq_unbound_cpumask_store() is for changing the cpumask of *all* workqueues. I don't think it can be used to make scsi and iscsi workqueues bound to different cpu.

apply_workqueue_attrs() is for changing the cpumask of the specific workqueue, which can change the cpumask of __WQ_ORDERED workqueue (but without __WQ_ORDERED_EXPLICIT).

Quoted 33 lines

5280 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);

     }

Thanks for your review.
Bob

Quoted 24 lines

Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.

Quoted 21 lines
include/linux/workqueue.h +0 -0
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -2
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
         * on NUMA.
         */
        if ((flags & WQ_UNBOUND) && max_active == 1)
-               flags |= __WQ_ORDERED;
+               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
+               if (!(flags & __WQ_ORDERED_DISABLE))
+                       flags |= __WQ_ORDERED;
 
        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
--

2.9.5

author
#8 lore raw
Quoted 63 lines · Lai Jiangshan
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

Quoted 26 lines

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +1 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
        __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
        __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
        __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
+       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
        WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
        WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)                            \
        alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
Quoted 7 lines

include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439

Quoted 3 lines
include/linux/workqueue.h +3 -0
+#define create_singlethread_workqueue_noorder(name)                    \
+       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.

I don't think so, see function workqueue_apply_unbound_cpumask():

wq_unbound_cpumask_store()

Quoted 8 lines
Quoted 2 lines

workqueue_set_unbound_cpumask()
workqueue_apply_unbound_cpumask() {

     ...

5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags & __WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only __WQ_ORDERED is set.

Quoted 2 lines

wq_unbound_cpumask_store() is for changing the cpumask of *all* workqueues.

Isn't '/sys/bus/workqueue/devices/xxxx/cpumask' using the same function to change cpumask of specific workqueue?
Am I missing something..

Quoted 3 lines

I don't think it can be used to make
scsi and iscsi workqueues bound to different cpu.

The idea is to register scsi/iscsi workqueues with WQ_SYSFS, and then they can be bounded to different cpu by writing cpu number to "/sys/bus/workqueue/devices/xxxx/cpumask".

Quoted 37 lines

apply_workqueue_attrs() is for changing the cpumask of the specific workqueue, which can change the cpumask of __WQ_ORDERED workqueue (but without __WQ_ORDERED_EXPLICIT).

Quoted 33 lines

5280 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);

     }

Thanks for your review.
Bob

Quoted 24 lines

Just use alloc_workqueue() with __WQ_ORDERED and max_active=1. It can
be wrapped as a new function or macro, but I don't think> create_singlethread_workqueue_noorder() is a good name for it.

Quoted 21 lines
include/linux/workqueue.h +0 -0
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
 struct workqueue_attrs *alloc_workqueue_attrs(void);
kernel/workqueue.c +3 -2
index 4e01c44..2167013 100644
@@ -4237,7 +4237,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
         * on NUMA.
         */
        if ((flags & WQ_UNBOUND) && max_active == 1)
-               flags |= __WQ_ORDERED;
+               /* the caller may don't want __WQ_ORDERED to be set implicitly. */
+               if (!(flags & __WQ_ORDERED_DISABLE))
+                       flags |= __WQ_ORDERED;
 
        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
--

2.9.5

author
#9 lore raw
In reply to Lai Jiangshan
Quoted 63 lines · Lai Jiangshan
Quoted 53 lines · Bob Liu
Quoted 40 lines · Lai Jiangshan
Quoted 7 lines · Bob Liu

Current code always set 'Unbound && max_active == 1' workqueues to ordered implicitly, while this may be not an expected behaviour for some use cases.

E.g some scsi and iscsi workqueues(unbound && max_active = 1) want to be bind to different cpu so as to get better isolation, but their cpumask can't be changed because WQ_ORDERED is set implicitly.

Hello

If I read the code correctly, the reason why their cpumask can't be changed is because __WQ_ORDERED_EXPLICIT, not __WQ_ORDERED.

Quoted 26 lines

This patch adds a flag __WQ_ORDERED_DISABLE and also create_singlethread_workqueue_noorder() to offer an new option.

---
 include/linux/workqueue.h | 4 ++++
 kernel/workqueue.c        | 4 +++-
 2 files changed, 7 insertions(+), 1 deletion(-)
include/linux/workqueue.h +1 -0
index e48554e..4c86913 100644
@@ -344,6 +344,7 @@ enum {
        __WQ_ORDERED            = 1 << 17, /* internal: workqueue is ordered */
        __WQ_LEGACY             = 1 << 18, /* internal: create*_workqueue() */
        __WQ_ORDERED_EXPLICIT   = 1 << 19, /* internal: alloc_ordered_workqueue() */
+       __WQ_ORDERED_DISABLE    = 1 << 20, /* internal: don't set __WQ_ORDERED implicitly */
 
        WQ_MAX_ACTIVE           = 512,    /* I like 512, better ideas? */
        WQ_MAX_UNBOUND_PER_CPU  = 4,      /* 4 * #cpus for unbound wq */
@@ -433,6 +434,9 @@ struct workqueue_struct *alloc_workqueue(const char *fmt,
 #define create_singlethread_workqueue(name)                            \
        alloc_ordered_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM, name)
 
Quoted 7 lines

include/linux/workqueue.h struct workqueue_struct *alloc_workqueue(const char *fmt, · line 439

Quoted 3 lines
include/linux/workqueue.h +3 -0
+#define create_singlethread_workqueue_noorder(name)                    \
+       alloc_workqueue("%s", WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | \
+                       WQ_UNBOUND | __WQ_ORDERED_DISABLE, 1, (name))

I think using __WQ_ORDERED without __WQ_ORDERED_EXPLICIT is what you need, in which case cpumask is allowed to be changed.

I don't think so, see function workqueue_apply_unbound_cpumask():

wq_unbound_cpumask_store()

Quoted 8 lines
Quoted 2 lines

workqueue_set_unbound_cpumask()
workqueue_apply_unbound_cpumask() {

     ...

5276 /* creating multiple pwqs breaks ordering guarantee */
5277 if (wq->flags & __WQ_ORDERED)
5278 continue;
^^^^
Here will skip apply cpumask if only __WQ_ORDERED is set.

Quoted 8 lines

wq_unbound_cpumask_store() is for changing the cpumask of *all* workqueues. I don't think it can be used to make scsi and iscsi workqueues bound to different cpu.

apply_workqueue_attrs() is for changing the cpumask of the specific workqueue, which can change the cpumask of __WQ_ORDERED workqueue (but without __WQ_ORDERED_EXPLICIT).

Yes, you are right. I made a mistake.
Sorry for the noise.

Regards,
Bob

Quoted 4 lines
Quoted 4 lines

5280 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);

     }