6.8.4-rc1 review

@@ -7186,7 +7186,7 @@ void __init workqueue_init_early(void)
 					      WQ_FREEZABLE, 0);
 	system_power_efficient_wq = alloc_workqueue("events_power_efficient",
 					      WQ_POWER_EFFICIENT, 0);
-	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
+	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
 					      WQ_FREEZABLE | WQ_POWER_EFFICIENT,
 					      0);
 	BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||

@@ -1506,7 +1506,7 @@ static void wq_update_node_max_active(st
 
 	lockdep_assert_held(&wq->mutex);
 
-	if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
+	if (!cpumask_test_cpu(off_cpu, effective))
 		off_cpu = -1;
 
 	total_cpus = cpumask_weight_and(effective, cpu_online_mask);

@@ -391,13 +391,6 @@ enum {
 	WQ_MAX_ACTIVE		= 512,	  /* I like 512, better ideas? */
 	WQ_UNBOUND_MAX_ACTIVE	= WQ_MAX_ACTIVE,
 	WQ_DFL_ACTIVE		= WQ_MAX_ACTIVE / 2,
-
-	/*
-	 * Per-node default cap on min_active. Unless explicitly set, min_active
-	 * is set to min(max_active, WQ_DFL_MIN_ACTIVE). For more details, see
-	 * workqueue_struct->min_active definition.
-	 */
-	WQ_DFL_MIN_ACTIVE	= 8,
 };
 
 /*

@@ -440,33 +433,11 @@ extern struct workqueue_struct *system_f
  * alloc_workqueue - allocate a workqueue
  * @fmt: printf format for the name of the workqueue
  * @flags: WQ_* flags
- * @max_active: max in-flight work items, 0 for default
+ * @max_active: max in-flight work items per CPU, 0 for default
  * remaining args: args for @fmt
  *
- * For a per-cpu workqueue, @max_active limits the number of in-flight work
- * items for each CPU. e.g. @max_active of 1 indicates that each CPU can be
- * executing at most one work item for the workqueue.
- *
- * For unbound workqueues, @max_active limits the number of in-flight work items
- * for the whole system. e.g. @max_active of 16 indicates that that there can be
- * at most 16 work items executing for the workqueue in the whole system.
- *
- * As sharing the same active counter for an unbound workqueue across multiple
- * NUMA nodes can be expensive, @max_active is distributed to each NUMA node
- * according to the proportion of the number of online CPUs and enforced
- * independently.
- *
- * Depending on online CPU distribution, a node may end up with per-node
- * max_active which is significantly lower than @max_active, which can lead to
- * deadlocks if the per-node concurrency limit is lower than the maximum number
- * of interdependent work items for the workqueue.
- *
- * To guarantee forward progress regardless of online CPU distribution, the
- * concurrency limit on every node is guaranteed to be equal to or greater than
- * min_active which is set to min(@max_active, %WQ_DFL_MIN_ACTIVE). This means
- * that the sum of per-node max_active's may be larger than @max_active.
- *
- * For detailed information on %WQ_* flags, please refer to
+ * Allocate a workqueue with the specified parameters.  For detailed
+ * information on WQ_* flags, please refer to
  * Documentation/core-api/workqueue.rst.
  *
  * RETURNS:

540 lines not on this page · raw message

@@ -281,16 +281,6 @@ struct wq_flusher {
 struct wq_device;
 
 /*
- * Unlike in a per-cpu workqueue where max_active limits its concurrency level
- * on each CPU, in an unbound workqueue, max_active applies to the whole system.
- * As sharing a single nr_active across multiple sockets can be very expensive,
- * the counting and enforcement is per NUMA node.
- */
-struct wq_node_nr_active {
-	atomic_t		nr;		/* per-node nr_active count */
-};
-
-/*
  * The externally visible workqueue.  It relays the issued work items to
  * the appropriate worker_pool through its pool_workqueues.
  */

@@ -336,7 +326,6 @@ struct workqueue_struct {
 	/* hot fields used during command issue, aligned to cacheline */
 	unsigned int		flags ____cacheline_aligned; /* WQ: WQ_* flags */
 	struct pool_workqueue __percpu __rcu **cpu_pwq; /* I: per-cpu pwqs */
-	struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
 };
 
 static struct kmem_cache *pwq_cache;

@@ -1433,31 +1422,6 @@ work_func_t wq_worker_last_func(struct t
 }
 
 /**
- * wq_node_nr_active - Determine wq_node_nr_active to use
- * @wq: workqueue of interest
- * @node: NUMA node, can be %NUMA_NO_NODE
- *
- * Determine wq_node_nr_active to use for @wq on @node. Returns:
- *
- * - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
- *
- * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
- *
- * - Otherwise, node_nr_active[@node].
- */
-static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
-						   int node)
-{
-	if (!(wq->flags & WQ_UNBOUND))
-		return NULL;
-
-	if (node == NUMA_NO_NODE)
-		node = nr_node_ids;
-
-	return wq->node_nr_active[node];
-}
-
-/**
  * get_pwq - get an extra reference on the specified pool_workqueue
  * @pwq: pool_workqueue to get
  *

@@ -1538,17 +1502,12 @@ static bool pwq_activate_work(struct poo
 			      struct work_struct *work)
 {
 	struct worker_pool *pool = pwq->pool;
-	struct wq_node_nr_active *nna;
 
 	lockdep_assert_held(&pool->lock);
 
 	if (!(*work_data_bits(work) & WORK_STRUCT_INACTIVE))
 		return false;
 
-	nna = wq_node_nr_active(pwq->wq, pool->node);
-	if (nna)
-		atomic_inc(&nna->nr);
-
 	pwq->nr_active++;
 	__pwq_activate_work(pwq, work);
 	return true;

@@ -1565,18 +1524,14 @@ static bool pwq_tryinc_nr_active(struct
 {
 	struct workqueue_struct *wq = pwq->wq;
 	struct worker_pool *pool = pwq->pool;
-	struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
 	bool obtained;
 
 	lockdep_assert_held(&pool->lock);
 
 	obtained = pwq->nr_active < READ_ONCE(wq->max_active);
 
-	if (obtained) {
+	if (obtained)
 		pwq->nr_active++;
-		if (nna)
-			atomic_inc(&nna->nr);
-	}
 	return obtained;
 }
 

@@ -1613,26 +1568,10 @@ static bool pwq_activate_first_inactive(
 static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 {
 	struct worker_pool *pool = pwq->pool;
-	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);
 
 	lockdep_assert_held(&pool->lock);
 
-	/*
-	 * @pwq->nr_active should be decremented for both percpu and unbound
-	 * workqueues.
-	 */
 	pwq->nr_active--;
-
-	/*
-	 * For a percpu workqueue, it's simple. Just need to kick the first
-	 * inactive work item on @pwq itself.
-	 */
-	if (!nna) {
-		pwq_activate_first_inactive(pwq);
-		return;
-	}
-
-	atomic_dec(&nna->nr);
 	pwq_activate_first_inactive(pwq);
 }
 

@@ -4087,63 +4026,11 @@ static void wq_free_lockdep(struct workq
 }
 #endif
 
-static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
-{
-	int node;
-
-	for_each_node(node) {
-		kfree(nna_ar[node]);
-		nna_ar[node] = NULL;
-	}
-
-	kfree(nna_ar[nr_node_ids]);
-	nna_ar[nr_node_ids] = NULL;
-}
-
-static void init_node_nr_active(struct wq_node_nr_active *nna)
-{
-	atomic_set(&nna->nr, 0);
-}
-
-/*
- * Each node's nr_active counter will be accessed mostly from its own node and
- * should be allocated in the node.
- */
-static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
-{
-	struct wq_node_nr_active *nna;
-	int node;
-
-	for_each_node(node) {
-		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
-		if (!nna)
-			goto err_free;
-		init_node_nr_active(nna);
-		nna_ar[node] = nna;
-	}
-
-	/* [nr_node_ids] is used as the fallback */
-	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
-	if (!nna)
-		goto err_free;
-	init_node_nr_active(nna);
-	nna_ar[nr_node_ids] = nna;
-
-	return 0;
-
-err_free:
-	free_node_nr_active(nna_ar);
-	return -ENOMEM;
-}
-
 static void rcu_free_wq(struct rcu_head *rcu)
 {
 	struct workqueue_struct *wq =
 		container_of(rcu, struct workqueue_struct, rcu);
 
-	if (wq->flags & WQ_UNBOUND)
-		free_node_nr_active(wq->node_nr_active);
-
 	wq_free_lockdep(wq);
 	free_percpu(wq->cpu_pwq);
 	free_workqueue_attrs(wq->unbound_attrs);

@@ -4882,8 +4769,7 @@ struct workqueue_struct *alloc_workqueue
 {
 	va_list args;
 	struct workqueue_struct *wq;
-	size_t wq_size;
-	int name_len;
+	int len;
 
 	/*
 	 * Unbound && max_active == 1 used to imply ordered, which is no longer

@@ -4899,12 +4785,7 @@ struct workqueue_struct *alloc_workqueue
 		flags |= WQ_UNBOUND;
 
 	/* allocate wq and format name */
-	if (flags & WQ_UNBOUND)
-		wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
-	else
-		wq_size = sizeof(*wq);
-
-	wq = kzalloc(wq_size, GFP_KERNEL);
+	wq = kzalloc(sizeof(*wq), GFP_KERNEL);
 	if (!wq)
 		return NULL;
 

@@ -4915,12 +4796,11 @@ struct workqueue_struct *alloc_workqueue
 	}
 
 	va_start(args, max_active);
-	name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
+	len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
 	va_end(args);
 
-	if (name_len >= WQ_NAME_LEN)
-		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
-			     wq->name);
+	if (len >= WQ_NAME_LEN)
+		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n", wq->name);
 
 	max_active = max_active ?: WQ_DFL_ACTIVE;
 	max_active = wq_clamp_max_active(max_active, flags, wq->name);

@@ -4939,13 +4819,8 @@ struct workqueue_struct *alloc_workqueue
 	wq_init_lockdep(wq);
 	INIT_LIST_HEAD(&wq->list);
 
-	if (flags & WQ_UNBOUND) {
-		if (alloc_node_nr_active(wq->node_nr_active) < 0)
-			goto err_unreg_lockdep;
-	}
-
 	if (alloc_and_link_pwqs(wq) < 0)
-		goto err_free_node_nr_active;
+		goto err_unreg_lockdep;
 
 	if (wq_online && init_rescuer(wq) < 0)
 		goto err_destroy;

@@ -4970,9 +4845,6 @@ struct workqueue_struct *alloc_workqueue
 
 	return wq;
 
-err_free_node_nr_active:
-	if (wq->flags & WQ_UNBOUND)
-		free_node_nr_active(wq->node_nr_active);
 err_unreg_lockdep:
 	wq_unregister_lockdep(wq);
 	wq_free_lockdep(wq);

@@ -304,7 +304,7 @@ struct workqueue_struct {
 	int			saved_max_active; /* WQ: saved max_active */
 
 	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */
-	struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */
+	struct pool_workqueue	*dfl_pwq;	/* PW: only for unbound wqs */
 
 #ifdef CONFIG_SYSFS
 	struct wq_device	*wq_dev;	/* I: for sysfs interface */

@@ -635,23 +635,6 @@ static int worker_pool_assign_id(struct
 	return ret;
 }
 
-static struct pool_workqueue __rcu **
-unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
-{
-       if (cpu >= 0)
-               return per_cpu_ptr(wq->cpu_pwq, cpu);
-       else
-               return &wq->dfl_pwq;
-}
-
-/* @cpu < 0 for dfl_pwq */
-static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
-{
-	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
-				     lockdep_is_held(&wq_pool_mutex) ||
-				     lockdep_is_held(&wq->mutex));
-}
-
 static unsigned int work_color_to_flags(int color)
 {
 	return color << WORK_STRUCT_COLOR_SHIFT;

@@ -4341,11 +4324,10 @@ static void wq_calc_pod_cpumask(struct w
 				"possible intersect\n");
 }
 
-/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
+/* install @pwq into @wq's cpu_pwq and return the old pwq */
 static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
 					int cpu, struct pool_workqueue *pwq)
 {
-	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
 	struct pool_workqueue *old_pwq;
 
 	lockdep_assert_held(&wq_pool_mutex);

@@ -4354,8 +4336,8 @@ static struct pool_workqueue *install_un
 	/* link_pwq() can handle duplicate calls */
 	link_pwq(pwq);
 
-	old_pwq = rcu_access_pointer(*slot);
-	rcu_assign_pointer(*slot, pwq);
+	old_pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
+	rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
 	return old_pwq;
 }
 

@@ -4455,11 +4437,14 @@ static void apply_wqattrs_commit(struct
 
 	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
 
-	/* save the previous pwqs and install the new ones */
+	/* save the previous pwq and install the new one */
 	for_each_possible_cpu(cpu)
 		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
 							ctx->pwq_tbl[cpu]);
-	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
+
+	/* @dfl_pwq might not have been used, ensure it's linked */
+	link_pwq(ctx->dfl_pwq);
+	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
 
 	mutex_unlock(&ctx->wq->mutex);
 }

@@ -4569,7 +4554,9 @@ static void wq_update_pod(struct workque
 
 	/* nothing to do if the target cpumask matches the current pwq */
 	wq_calc_pod_cpumask(target_attrs, cpu, off_cpu);
-	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
+	pwq = rcu_dereference_protected(*per_cpu_ptr(wq->cpu_pwq, cpu),
+					lockdep_is_held(&wq_pool_mutex));
+	if (wqattrs_equal(target_attrs, pwq->pool->attrs))
 		return;
 
 	/* create a new pwq */

@@ -4587,11 +4574,10 @@ static void wq_update_pod(struct workque
 
 use_dfl_pwq:
 	mutex_lock(&wq->mutex);
-	pwq = unbound_pwq(wq, -1);
-	raw_spin_lock_irq(&pwq->pool->lock);
-	get_pwq(pwq);
-	raw_spin_unlock_irq(&pwq->pool->lock);
-	old_pwq = install_unbound_pwq(wq, cpu, pwq);
+	raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
+	get_pwq(wq->dfl_pwq);
+	raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
+	old_pwq = install_unbound_pwq(wq, cpu, wq->dfl_pwq);
 out_unlock:
 	mutex_unlock(&wq->mutex);
 	put_pwq_unlocked(old_pwq);

@@ -4629,13 +4615,10 @@ static int alloc_and_link_pwqs(struct wo
 
 	cpus_read_lock();
 	if (wq->flags & __WQ_ORDERED) {
-		struct pool_workqueue *dfl_pwq;
-
 		ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
 		/* there should only be single pwq for ordering guarantee */
-		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
-		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
-			      wq->pwqs.prev != &dfl_pwq->pwqs_node),
+		WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
+			      wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
 		     "ordering guarantee broken for workqueue %s\n", wq->name);
 	} else {
 		ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);

@@ -4866,7 +4849,7 @@ static bool pwq_busy(struct pool_workque
 		if (pwq->nr_in_flight[i])
 			return true;
 
-	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
+	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
 		return true;
 	if (!pwq_is_empty(pwq))
 		return true;

@@ -4950,12 +4933,13 @@ void destroy_workqueue(struct workqueue_
 	rcu_read_lock();
 
 	for_each_possible_cpu(cpu) {
-		put_pwq_unlocked(unbound_pwq(wq, cpu));
-		RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
+		pwq = rcu_access_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu));
+		RCU_INIT_POINTER(*per_cpu_ptr(wq->cpu_pwq, cpu), NULL);
+		put_pwq_unlocked(pwq);
 	}
 
-	put_pwq_unlocked(unbound_pwq(wq, -1));
-	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
+	put_pwq_unlocked(wq->dfl_pwq);
+	wq->dfl_pwq = NULL;
 
 	rcu_read_unlock();
 }

@@ -4703,7 +4703,7 @@ static int init_rescuer(struct workqueue
  */
 static void wq_adjust_max_active(struct workqueue_struct *wq)
 {
-	bool activated;
+	struct pool_workqueue *pwq;
 
 	lockdep_assert_held(&wq->mutex);
 

@@ -4723,26 +4723,19 @@ static void wq_adjust_max_active(struct
 	 */
 	WRITE_ONCE(wq->max_active, wq->saved_max_active);
 
-	/*
-	 * Round-robin through pwq's activating the first inactive work item
-	 * until max_active is filled.
-	 */
-	do {
-		struct pool_workqueue *pwq;
+	for_each_pwq(pwq, wq) {
+		unsigned long flags;
+
+		/* this function can be called during early boot w/ irq disabled */
+		raw_spin_lock_irqsave(&pwq->pool->lock, flags);
 
-		activated = false;
-		for_each_pwq(pwq, wq) {
-			unsigned long flags;
-
-			/* can be called during early boot w/ irq disabled */
-			raw_spin_lock_irqsave(&pwq->pool->lock, flags);
-			if (pwq_activate_first_inactive(pwq)) {
-				activated = true;
-				kick_pool(pwq->pool);
-			}
-			raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
-		}
-	} while (activated);
+		while (pwq_activate_first_inactive(pwq))
+			;
+
+		kick_pool(pwq->pool);
+
+		raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
+	}
 }
 
 __printf(1, 4)

@@ -1464,14 +1464,11 @@ static bool pwq_is_empty(struct pool_wor
 static void __pwq_activate_work(struct pool_workqueue *pwq,
 				struct work_struct *work)
 {
-	unsigned long *wdb = work_data_bits(work);
-
-	WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
 	trace_workqueue_activate_work(work);
 	if (list_empty(&pwq->pool->worklist))
 		pwq->pool->watchdog_ts = jiffies;
 	move_linked_works(work, &pwq->pool->worklist, NULL);
-	__clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
+	__clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
 }
 
 /**

@@ -1496,66 +1493,12 @@ static bool pwq_activate_work(struct poo
 	return true;
 }
 
-/**
- * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
- * @pwq: pool_workqueue of interest
- *
- * Try to increment nr_active for @pwq. Returns %true if an nr_active count is
- * successfully obtained. %false otherwise.
- */
-static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq)
-{
-	struct workqueue_struct *wq = pwq->wq;
-	struct worker_pool *pool = pwq->pool;
-	bool obtained;
-
-	lockdep_assert_held(&pool->lock);
-
-	obtained = pwq->nr_active < READ_ONCE(wq->max_active);
-
-	if (obtained)
-		pwq->nr_active++;
-	return obtained;
-}
-
-/**
- * pwq_activate_first_inactive - Activate the first inactive work item on a pwq
- * @pwq: pool_workqueue of interest
- *
- * Activate the first inactive work item of @pwq if available and allowed by
- * max_active limit.
- *
- * Returns %true if an inactive work item has been activated. %false if no
- * inactive work item is found or max_active limit is reached.
- */
-static bool pwq_activate_first_inactive(struct pool_workqueue *pwq)
-{
-	struct work_struct *work =
-		list_first_entry_or_null(&pwq->inactive_works,
-					 struct work_struct, entry);
-
-	if (work && pwq_tryinc_nr_active(pwq)) {
-		__pwq_activate_work(pwq, work);
-		return true;
-	} else {
-		return false;
-	}
-}
-
-/**
- * pwq_dec_nr_active - Retire an active count
- * @pwq: pool_workqueue of interest
- *
- * Decrement @pwq's nr_active and try to activate the first inactive work item.
- */
-static void pwq_dec_nr_active(struct pool_workqueue *pwq)
+static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
 {
-	struct worker_pool *pool = pwq->pool;
+	struct work_struct *work = list_first_entry(&pwq->inactive_works,
+						    struct work_struct, entry);
 
-	lockdep_assert_held(&pool->lock);
-
-	pwq->nr_active--;
-	pwq_activate_first_inactive(pwq);
+	pwq_activate_work(pwq, work);
 }
 
 /**

@@ -1573,8 +1516,14 @@ static void pwq_dec_nr_in_flight(struct
 {
 	int color = get_work_color(work_data);
 
-	if (!(work_data & WORK_STRUCT_INACTIVE))
-		pwq_dec_nr_active(pwq);
+	if (!(work_data & WORK_STRUCT_INACTIVE)) {
+		pwq->nr_active--;
+		if (!list_empty(&pwq->inactive_works)) {
+			/* one down, submit an inactive one */
+			if (pwq->nr_active < READ_ONCE(pwq->wq->max_active))
+				pwq_activate_first_inactive(pwq);
+		}
+	}
 
 	pwq->nr_in_flight[color]--;
 

@@ -1876,11 +1825,13 @@ retry:
 	 * @work must also queue behind existing inactive work items to maintain
 	 * ordering when max_active changes. See wq_adjust_max_active().
 	 */
-	if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq)) {
+	if (list_empty(&pwq->inactive_works) &&
+	    pwq->nr_active < READ_ONCE(pwq->wq->max_active)) {
 		if (list_empty(&pool->worklist))
 			pool->watchdog_ts = jiffies;
 
 		trace_workqueue_activate_work(work);
+		pwq->nr_active++;
 		insert_work(pwq, work, &pool->worklist, work_flags);
 		kick_pool(pool);
 	} else {

@@ -4729,8 +4680,9 @@ static void wq_adjust_max_active(struct
 		/* this function can be called during early boot w/ irq disabled */
 		raw_spin_lock_irqsave(&pwq->pool->lock, flags);
 
-		while (pwq_activate_first_inactive(pwq))
-			;
+		while (!list_empty(&pwq->inactive_works) &&
+		       pwq->nr_active < wq->max_active)
+			pwq_activate_first_inactive(pwq);
 
 		kick_pool(pwq->pool);
 

@@ -1461,36 +1461,16 @@ static bool pwq_is_empty(struct pool_wor
 	return !pwq->nr_active && list_empty(&pwq->inactive_works);
 }
 
-static void __pwq_activate_work(struct pool_workqueue *pwq,
-				struct work_struct *work)
+static void pwq_activate_inactive_work(struct work_struct *work)
 {
+	struct pool_workqueue *pwq = get_work_pwq(work);
+
 	trace_workqueue_activate_work(work);
 	if (list_empty(&pwq->pool->worklist))
 		pwq->pool->watchdog_ts = jiffies;
 	move_linked_works(work, &pwq->pool->worklist, NULL);
 	__clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
-}
-
-/**
- * pwq_activate_work - Activate a work item if inactive
- * @pwq: pool_workqueue @work belongs to
- * @work: work item to activate
- *
- * Returns %true if activated. %false if already active.
- */
-static bool pwq_activate_work(struct pool_workqueue *pwq,
-			      struct work_struct *work)
-{
-	struct worker_pool *pool = pwq->pool;
-
-	lockdep_assert_held(&pool->lock);
-
-	if (!(*work_data_bits(work) & WORK_STRUCT_INACTIVE))
-		return false;
-
 	pwq->nr_active++;
-	__pwq_activate_work(pwq, work);
-	return true;
 }
 
 static void pwq_activate_first_inactive(struct pool_workqueue *pwq)

@@ -1498,7 +1478,7 @@ static void pwq_activate_first_inactive(
 	struct work_struct *work = list_first_entry(&pwq->inactive_works,
 						    struct work_struct, entry);
 
-	pwq_activate_work(pwq, work);
+	pwq_activate_inactive_work(work);
 }
 
 /**

@@ -1636,7 +1616,8 @@ static int try_to_grab_pending(struct wo
 		 * management later on and cause stall.  Make sure the work
 		 * item is activated before grabbing.
 		 */
-		pwq_activate_work(pwq, work);
+		if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
+			pwq_activate_inactive_work(work);
 
 		list_del_init(&work->entry);
 		pwq_dec_nr_in_flight(pwq, *work_data_bits(work));

@@ -1456,11 +1456,6 @@ static void put_pwq_unlocked(struct pool
 	}
 }
 
-static bool pwq_is_empty(struct pool_workqueue *pwq)
-{
-	return !pwq->nr_active && list_empty(&pwq->inactive_works);
-}
-
 static void pwq_activate_inactive_work(struct work_struct *work)
 {
 	struct pool_workqueue *pwq = get_work_pwq(work);

@@ -3330,7 +3325,7 @@ reflush:
 		bool drained;
 
 		raw_spin_lock_irq(&pwq->pool->lock);
-		drained = pwq_is_empty(pwq);
+		drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
 		raw_spin_unlock_irq(&pwq->pool->lock);
 
 		if (drained)

@@ -4777,7 +4772,7 @@ static bool pwq_busy(struct pool_workque
 
 	if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
 		return true;
-	if (!pwq_is_empty(pwq))
+	if (pwq->nr_active || !list_empty(&pwq->inactive_works))
 		return true;
 
 	return false;

@@ -5215,7 +5210,7 @@ void show_one_workqueue(struct workqueue
 	unsigned long flags;
 
 	for_each_pwq(pwq, wq) {
-		if (!pwq_is_empty(pwq)) {
+		if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
 			idle = false;
 			break;
 		}

@@ -5227,7 +5222,7 @@ void show_one_workqueue(struct workqueue
 
 	for_each_pwq(pwq, wq) {
 		raw_spin_lock_irqsave(&pwq->pool->lock, flags);
-		if (!pwq_is_empty(pwq)) {
+		if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
 			/*
 			 * Defer printing to avoid deadlocks in console
 			 * drivers that queue work while holding locks

@@ -143,9 +143,6 @@ enum {
  *
  * WR: wq->mutex protected for writes.  RCU protected for reads.
  *
- * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
- *     with READ_ONCE() without locking.
- *
  * MD: wq_mayday_lock protected.
  *
  * WD: Used internally by the watchdog.

@@ -253,6 +250,7 @@ struct pool_workqueue {
 	 * is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
 	 */
 	int			nr_active;	/* L: nr of active works */
+	int			max_active;	/* L: max active works */
 	struct list_head	inactive_works;	/* L: inactive works */
 	struct list_head	pwqs_node;	/* WR: node on wq->pwqs */
 	struct list_head	mayday_node;	/* MD: node on wq->maydays */

@@ -300,8 +298,7 @@ struct workqueue_struct {
 	struct worker		*rescuer;	/* MD: rescue worker */
 
 	int			nr_drainers;	/* WQ: drain in progress */
-	int			max_active;	/* WO: max active works */
-	int			saved_max_active; /* WQ: saved max_active */
+	int			saved_max_active; /* WQ: saved pwq max_active */
 
 	struct workqueue_attrs	*unbound_attrs;	/* PW: only for unbound wqs */
 	struct pool_workqueue	*dfl_pwq;	/* PW: only for unbound wqs */

@@ -1495,7 +1492,7 @@ static void pwq_dec_nr_in_flight(struct
 		pwq->nr_active--;
 		if (!list_empty(&pwq->inactive_works)) {
 			/* one down, submit an inactive one */
-			if (pwq->nr_active < READ_ONCE(pwq->wq->max_active))
+			if (pwq->nr_active < pwq->max_active)
 				pwq_activate_first_inactive(pwq);
 		}
 	}

@@ -1796,13 +1793,7 @@ retry:
 	pwq->nr_in_flight[pwq->work_color]++;
 	work_flags = work_color_to_flags(pwq->work_color);
 
-	/*
-	 * Limit the number of concurrently active work items to max_active.
-	 * @work must also queue behind existing inactive work items to maintain
-	 * ordering when max_active changes. See wq_adjust_max_active().
-	 */
-	if (list_empty(&pwq->inactive_works) &&
-	    pwq->nr_active < READ_ONCE(pwq->wq->max_active)) {
+	if (likely(pwq->nr_active < pwq->max_active)) {
 		if (list_empty(&pool->worklist))
 			pool->watchdog_ts = jiffies;
 

@@ -4151,6 +4142,50 @@ static void pwq_release_workfn(struct kt
 	}
 }
 
+/**
+ * pwq_adjust_max_active - update a pwq's max_active to the current setting
+ * @pwq: target pool_workqueue
+ *
+ * If @pwq isn't freezing, set @pwq->max_active to the associated
+ * workqueue's saved_max_active and activate inactive work items
+ * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero.
+ */
+static void pwq_adjust_max_active(struct pool_workqueue *pwq)
+{
+	struct workqueue_struct *wq = pwq->wq;
+	bool freezable = wq->flags & WQ_FREEZABLE;
+	unsigned long flags;
+
+	/* for @wq->saved_max_active */
+	lockdep_assert_held(&wq->mutex);
+
+	/* fast exit for non-freezable wqs */
+	if (!freezable && pwq->max_active == wq->saved_max_active)
+		return;
+
+	/* this function can be called during early boot w/ irq disabled */
+	raw_spin_lock_irqsave(&pwq->pool->lock, flags);
+
+	/*
+	 * During [un]freezing, the caller is responsible for ensuring that
+	 * this function is called at least once after @workqueue_freezing
+	 * is updated and visible.
+	 */
+	if (!freezable || !workqueue_freezing) {
+		pwq->max_active = wq->saved_max_active;
+
+		while (!list_empty(&pwq->inactive_works) &&
+		       pwq->nr_active < pwq->max_active)
+			pwq_activate_first_inactive(pwq);
+
+		kick_pool(pwq->pool);
+	} else {
+		pwq->max_active = 0;
+	}
+
+	raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
+}
+
 /* initialize newly allocated @pwq which is associated with @wq and @pool */
 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
 		     struct worker_pool *pool)

@@ -4183,6 +4218,9 @@ static void link_pwq(struct pool_workque
 	/* set the matching work_color */
 	pwq->work_color = wq->work_color;
 
+	/* sync max_active to the current setting */
+	pwq_adjust_max_active(pwq);
+
 	/* link in @pwq */
 	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
 }

@@ -4620,52 +4658,6 @@ static int init_rescuer(struct workqueue
 	return 0;
 }
 
-/**
- * wq_adjust_max_active - update a wq's max_active to the current setting
- * @wq: target workqueue
- *
- * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
- * activate inactive work items accordingly. If @wq is freezing, clear
- * @wq->max_active to zero.
- */
-static void wq_adjust_max_active(struct workqueue_struct *wq)
-{
-	struct pool_workqueue *pwq;
-
-	lockdep_assert_held(&wq->mutex);
-
-	if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
-		WRITE_ONCE(wq->max_active, 0);
-		return;
-	}
-
-	if (wq->max_active == wq->saved_max_active)
-		return;
-
-	/*
-	 * Update @wq->max_active and then kick inactive work items if more
-	 * active work items are allowed. This doesn't break work item ordering
-	 * because new work items are always queued behind existing inactive
-	 * work items if there are any.
-	 */
-	WRITE_ONCE(wq->max_active, wq->saved_max_active);
-
-	for_each_pwq(pwq, wq) {
-		unsigned long flags;
-
-		/* this function can be called during early boot w/ irq disabled */
-		raw_spin_lock_irqsave(&pwq->pool->lock, flags);
-
-		while (!list_empty(&pwq->inactive_works) &&
-		       pwq->nr_active < wq->max_active)
-			pwq_activate_first_inactive(pwq);
-
-		kick_pool(pwq->pool);
-
-		raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
-	}
-}
-
 __printf(1, 4)
 struct workqueue_struct *alloc_workqueue(const char *fmt,
 					 unsigned int flags,

@@ -4673,6 +4665,7 @@ struct workqueue_struct *alloc_workqueue
 {
 	va_list args;
 	struct workqueue_struct *wq;
+	struct pool_workqueue *pwq;
 	int len;
 
 	/*

@@ -4711,7 +4704,6 @@ struct workqueue_struct *alloc_workqueue
 
 	/* init wq */
 	wq->flags = flags;
-	wq->max_active = max_active;
 	wq->saved_max_active = max_active;
 	mutex_init(&wq->mutex);
 	atomic_set(&wq->nr_pwqs_to_flush, 0);

@@ -4740,7 +4732,8 @@ struct workqueue_struct *alloc_workqueue
 	mutex_lock(&wq_pool_mutex);
 
 	mutex_lock(&wq->mutex);
-	wq_adjust_max_active(wq);
+	for_each_pwq(pwq, wq)
+		pwq_adjust_max_active(pwq);
 	mutex_unlock(&wq->mutex);
 
 	list_add_tail_rcu(&wq->list, &workqueues);

@@ -4878,6 +4871,8 @@ EXPORT_SYMBOL_GPL(destroy_workqueue);
  */
 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
 {
+	struct pool_workqueue *pwq;
+
 	/* disallow meddling with max_active for ordered workqueues */
 	if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
 		return;

@@ -4888,7 +4883,9 @@ void workqueue_set_max_active(struct wor
 
 	wq->flags &= ~__WQ_ORDERED;
 	wq->saved_max_active = max_active;
-	wq_adjust_max_active(wq);
+
+	for_each_pwq(pwq, wq)
+		pwq_adjust_max_active(pwq);
 
 	mutex_unlock(&wq->mutex);
 }

@@ -5135,8 +5132,8 @@ static void show_pwq(struct pool_workque
 	pr_info("  pwq %d:", pool->id);
 	pr_cont_pool_info(pool);
 
-	pr_cont(" active=%d refcnt=%d%s\n",
-		pwq->nr_active, pwq->refcnt,
+	pr_cont(" active=%d/%d refcnt=%d%s\n",
+		pwq->nr_active, pwq->max_active, pwq->refcnt,
 		!list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
 
 	hash_for_each(pool->busy_hash, bkt, worker, hentry) {

@@ -5684,6 +5681,7 @@ EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
 void freeze_workqueues_begin(void)
 {
 	struct workqueue_struct *wq;
+	struct pool_workqueue *pwq;
 
 	mutex_lock(&wq_pool_mutex);
 

@@ -5692,7 +5690,8 @@ void freeze_workqueues_begin(void)
 
 	list_for_each_entry(wq, &workqueues, list) {
 		mutex_lock(&wq->mutex);
-		wq_adjust_max_active(wq);
+		for_each_pwq(pwq, wq)
+			pwq_adjust_max_active(pwq);
 		mutex_unlock(&wq->mutex);
 	}
 

@@ -5757,6 +5756,7 @@ out_unlock:
 void thaw_workqueues(void)
 {
 	struct workqueue_struct *wq;
+	struct pool_workqueue *pwq;
 
 	mutex_lock(&wq_pool_mutex);
 

@@ -5768,7 +5768,8 @@ void thaw_workqueues(void)
 	/* restore max_active and repopulate worklist */
 	list_for_each_entry(wq, &workqueues, list) {
 		mutex_lock(&wq->mutex);
-		wq_adjust_max_active(wq);
+		for_each_pwq(pwq, wq)
+			pwq_adjust_max_active(pwq);
 		mutex_unlock(&wq->mutex);
 	}
 

@@ -108,7 +108,7 @@ enum {
 	RESCUER_NICE_LEVEL	= MIN_NICE,
 	HIGHPRI_NICE_LEVEL	= MIN_NICE,
 
-	WQ_NAME_LEN		= 32,
+	WQ_NAME_LEN		= 24,
 };
 
 /*

@@ -4666,7 +4666,6 @@ struct workqueue_struct *alloc_workqueue
 	va_list args;
 	struct workqueue_struct *wq;
 	struct pool_workqueue *pwq;
-	int len;
 
 	/*
 	 * Unbound && max_active == 1 used to imply ordered, which is no longer

@@ -4693,12 +4692,9 @@ struct workqueue_struct *alloc_workqueue
 	}
 
 	va_start(args, max_active);
-	len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);
+	vsnprintf(wq->name, sizeof(wq->name), fmt, args);
 	va_end(args);
 
-	if (len >= WQ_NAME_LEN)
-		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n", wq->name);
-
 	max_active = max_active ?: WQ_DFL_ACTIVE;
 	max_active = wq_clamp_max_active(max_active, flags, wq->name);