Skip to contentSkip to conversation

Revert of simple work, and backport workqueue rework

RFC PATCH New Steven Rostedt 1 participant 8 messages Sep 20, 2019

Subsystems (11)
VFS Block layer SPI Scheduler +7 more subsystems
lore
Share & tools
lore

Steven Rostedt

author reviewer
#0 lore raw

cover Revert of simple work, and backport workqueue rework

To be able to backport some of the fixes done in upstream RT, I need to revert the simple work code and incorporate the work queue rework that was done in rt-devel. I originally thought this crashed under stress test (as I reported in the stable meeting), but found that the system booted a different kernel that had known issues. 4.19-rt with these patches have passed all my tests. But before posting an rc1, I was hoping to get some people to look it over and perhaps even test it some to make sure there's not any issues here.

The reason I'm doing this is that this goes beyond the stable scope for 4.19-rt, but I believe it is still necessary.

-- Steve

Daniel Wagner (1):
thermal: Defer thermal wakups to threads

Sebastian Andrzej Siewior (3):
fs/aio: simple simple work
block: blk-mq: move blk_queue_usage_counter_release() into process context
workqueue: rework

Steven Rostedt (VMware) (3):
revert-aio
revert-thermal
revert-block

----

 block/blk-core.c                       |  10 +-
 drivers/block/loop.c                   |   2 +-
 drivers/spi/spi-rockchip.c             |   1 -
 drivers/thermal/x86_pkg_temp_thermal.c |  52 +-----
 fs/aio.c                               |  12 +-
 include/linux/blk-cgroup.h             |   2 +-
 include/linux/blkdev.h                 |   4 +-
 include/linux/interrupt.h              |   5 -
 include/linux/kthread-cgroup.h         |  17 --
 include/linux/kthread.h                |  15 +-
 include/linux/swait.h                  |  14 ++
 include/linux/workqueue.h              |   4 -
 init/main.c                            |   1 -
 kernel/irq/manage.c                    |  36 +---
 kernel/kthread.c                       |  14 --
 kernel/sched/core.c                    |   1 +
 kernel/time/hrtimer.c                  |  24 ---
 kernel/workqueue.c                     | 300 ++++++++++++++-------------------
 18 files changed, 168 insertions(+), 346 deletions(-)
 delete mode 100644 include/linux/kthread-cgroup.h
author reviewer
#1 lore raw

RFC PATCH 1/7 revert-aio

From: "Steven Rostedt (VMware)" <[email protected]>

revert: fs/aio: simple simple work

Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 fs/aio.c | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)
fs/aio.c +2 -13
index 16dcf8521c2c..911e23087dfb 100644
@@ -42,7 +42,6 @@
 #include <linux/ramfs.h>
 #include <linux/percpu-refcount.h>
 #include <linux/mount.h>
-#include <linux/swork.h>
 
 #include <asm/kmap_types.h>
 #include <linux/uaccess.h>

@@ -122,7 +121,6 @@ struct kioctx {
 	long			nr_pages;
 
 	struct rcu_work		free_rwork;	/* see free_ioctx() */
-	struct swork_event	free_swork;	/* see free_ioctx() */
 
 	/*
 	 * signals when all in-flight requests are done

@@ -267,7 +265,6 @@ static int __init aio_setup(void)
 		.mount		= aio_mount,
 		.kill_sb	= kill_anon_super,
 	};
-	BUG_ON(swork_get());
 	aio_mnt = kern_mount(&aio_fs);
 	if (IS_ERR(aio_mnt))
 		panic("Failed to create aio fs mount.");

@@ -609,9 +606,9 @@ static void free_ioctx_reqs(struct percpu_ref *ref)
  * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  * now it's safe to cancel any that need to be.
  */
-static void free_ioctx_users_work(struct swork_event *sev)
+static void free_ioctx_users(struct percpu_ref *ref)
 {
-	struct kioctx *ctx = container_of(sev, struct kioctx, free_swork);
+	struct kioctx *ctx = container_of(ref, struct kioctx, users);
 	struct aio_kiocb *req;
 
 	spin_lock_irq(&ctx->ctx_lock);

@@ -629,14 +626,6 @@ static void free_ioctx_users_work(struct swork_event *sev)
 	percpu_ref_put(&ctx->reqs);
 }
 
-static void free_ioctx_users(struct percpu_ref *ref)
-{
-	struct kioctx *ctx = container_of(ref, struct kioctx, users);
-
-	INIT_SWORK(&ctx->free_swork, free_ioctx_users_work);
-	swork_queue(&ctx->free_swork);
-}
-
 static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
 {
 	unsigned i, new_nr;
Signature · 1 line
2.20.1
author reviewer
#2 lore raw

RFC PATCH 2/7 fs/aio: simple simple work

In reply to Steven Rostedt

From: Sebastian Andrzej Siewior <[email protected]>

[ Upstream commit 1a142116f6435ef070ecebb66d2d599507c10601 ]

|BUG: sleeping function called from invalid context at kernel/locking/rtmutex.c:768
|in_atomic(): 1, irqs_disabled(): 0, pid: 26, name: rcuos/2
|2 locks held by rcuos/2/26:
| #0:  (rcu_callback){.+.+..}, at: [<ffffffff810b1a12>] rcu_nocb_kthread+0x1e2/0x380
| #1:  (rcu_read_lock_sched){.+.+..}, at: [<ffffffff812acd26>] percpu_ref_kill_rcu+0xa6/0x1c0
|Preemption disabled at:[<ffffffff810b1a93>] rcu_nocb_kthread+0x263/0x380
|Call Trace:
| [<ffffffff81582e9e>] dump_stack+0x4e/0x9c
| [<ffffffff81077aeb>] __might_sleep+0xfb/0x170
| [<ffffffff81589304>] rt_spin_lock+0x24/0x70
| [<ffffffff811c5790>] free_ioctx_users+0x30/0x130
| [<ffffffff812ace34>] percpu_ref_kill_rcu+0x1b4/0x1c0
| [<ffffffff810b1a93>] rcu_nocb_kthread+0x263/0x380
| [<ffffffff8106e046>] kthread+0xd6/0xf0
| [<ffffffff81591eec>] ret_from_fork+0x7c/0xb0

replace this preempt_disable() friendly swork.

Reported-By Mike Galbraith <[email protected]>
Suggested-by Benjamin LaHaise <[email protected]>
Signed-off-by Sebastian Andrzej Siewior <[email protected]>
Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 fs/aio.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
fs/aio.c +11 -2
index 911e23087dfb..0c613d805bf1 100644
@@ -121,6 +121,7 @@ struct kioctx {
 	long			nr_pages;
 
 	struct rcu_work		free_rwork;	/* see free_ioctx() */
+	struct kthread_work	free_kwork;	/* see free_ioctx() */
 
 	/*
 	 * signals when all in-flight requests are done

@@ -606,9 +607,9 @@ static void free_ioctx_reqs(struct percpu_ref *ref)
  * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  * now it's safe to cancel any that need to be.
  */
-static void free_ioctx_users(struct percpu_ref *ref)
+static void free_ioctx_users_work(struct kthread_work *work)
 {
-	struct kioctx *ctx = container_of(ref, struct kioctx, users);
+	struct kioctx *ctx = container_of(work, struct kioctx, free_kwork);
 	struct aio_kiocb *req;
 
 	spin_lock_irq(&ctx->ctx_lock);

@@ -626,6 +627,14 @@ static void free_ioctx_users(struct percpu_ref *ref)
 	percpu_ref_put(&ctx->reqs);
 }
 
+static void free_ioctx_users(struct percpu_ref *ref)
+{
+	struct kioctx *ctx = container_of(ref, struct kioctx, users);
+
+	kthread_init_work(&ctx->free_kwork, free_ioctx_users_work);
+	kthread_schedule_work(&ctx->free_kwork);
+}
+
 static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
 {
 	unsigned i, new_nr;
Signature · 1 line
2.20.1
author reviewer
#3 lore raw

RFC PATCH 3/7 revert-thermal

In reply to Steven Rostedt

From: "Steven Rostedt (VMware)" <[email protected]>

Revert: thermal: Defer thermal wakups to threads

Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 drivers/thermal/x86_pkg_temp_thermal.c | 52 ++------------------------
 1 file changed, 3 insertions(+), 49 deletions(-)
drivers/thermal/x86_pkg_temp_thermal.c +3 -49
index a5991cbb408f..1ef937d799e4 100644
@@ -29,7 +29,6 @@
 #include <linux/pm.h>
 #include <linux/thermal.h>
 #include <linux/debugfs.h>
-#include <linux/swork.h>
 #include <asm/cpu_device_id.h>
 #include <asm/mce.h>
 

@@ -330,7 +329,7 @@ static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
 	schedule_delayed_work_on(cpu, work, ms);
 }
 
-static void pkg_thermal_notify_work(struct swork_event *event)
+static int pkg_thermal_notify(u64 msr_val)
 {
 	int cpu = smp_processor_id();
 	struct pkg_device *pkgdev;

@@ -349,47 +348,9 @@ static void pkg_thermal_notify_work(struct swork_event *event)
 	}
 
 	spin_unlock_irqrestore(&pkg_temp_lock, flags);
-}
-
-#ifdef CONFIG_PREEMPT_RT_FULL
-static struct swork_event notify_work;
-
-static int pkg_thermal_notify_work_init(void)
-{
-	int err;
-
-	err = swork_get();
-	if (err)
-		return err;
-
-	INIT_SWORK(&notify_work, pkg_thermal_notify_work);
 	return 0;
 }
 
-static void pkg_thermal_notify_work_cleanup(void)
-{
-	swork_put();
-}
-
-static int pkg_thermal_notify(u64 msr_val)
-{
-	swork_queue(&notify_work);
-	return 0;
-}
-
-#else  /* !CONFIG_PREEMPT_RT_FULL */
-
-static int pkg_thermal_notify_work_init(void) { return 0; }
-
-static void pkg_thermal_notify_work_cleanup(void) {  }
-
-static int pkg_thermal_notify(u64 msr_val)
-{
-	pkg_thermal_notify_work(NULL);
-	return 0;
-}
-#endif /* CONFIG_PREEMPT_RT_FULL */
-
 static int pkg_temp_thermal_device_add(unsigned int cpu)
 {
 	int pkgid = topology_logical_package_id(cpu);

@@ -554,16 +515,11 @@ static int __init pkg_temp_thermal_init(void)
 	if (!x86_match_cpu(pkg_temp_thermal_ids))
 		return -ENODEV;
 
-	if (!pkg_thermal_notify_work_init())
-		return -ENODEV;
-
 	max_packages = topology_max_packages();
 	packages = kcalloc(max_packages, sizeof(struct pkg_device *),
 			   GFP_KERNEL);
-	if (!packages) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	if (!packages)
+		return -ENOMEM;
 
 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
 				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);

@@ -581,7 +537,6 @@ static int __init pkg_temp_thermal_init(void)
 	return 0;
 
 err:
-	pkg_thermal_notify_work_cleanup();
 	kfree(packages);
 	return ret;
 }

@@ -595,7 +550,6 @@ static void __exit pkg_temp_thermal_exit(void)
 	cpuhp_remove_state(pkg_thermal_hp_state);
 	debugfs_remove_recursive(debugfs);
 	kfree(packages);
-	pkg_thermal_notify_work_cleanup();
 }
 module_exit(pkg_temp_thermal_exit)
Signature · 1 line
2.20.1
author reviewer
#4 lore raw

RFC PATCH 4/7 thermal: Defer thermal wakups to threads

In reply to Steven Rostedt

From: Daniel Wagner <[email protected]>

[ Upstream commit ad2408dc248fe58536eef5b2b5734d8f9d3a280b ]

On RT the spin lock in pkg_temp_thermal_platfrom_thermal_notify will call schedule while we run in irq context.

[<ffffffff816850ac>] dump_stack+0x4e/0x8f
[<ffffffff81680f7d>] __schedule_bug+0xa6/0xb4
[<ffffffff816896b4>] __schedule+0x5b4/0x700
[<ffffffff8168982a>] schedule+0x2a/0x90
[<ffffffff8168a8b5>] rt_spin_lock_slowlock+0xe5/0x2d0
[<ffffffff8168afd5>] rt_spin_lock+0x25/0x30
[<ffffffffa03a7b75>] pkg_temp_thermal_platform_thermal_notify+0x45/0x134 [x86_pkg_temp_thermal]
[<ffffffff8103d4db>] ? therm_throt_process+0x1b/0x160
[<ffffffff8103d831>] intel_thermal_interrupt+0x211/0x250
[<ffffffff8103d8c1>] smp_thermal_interrupt+0x21/0x40
[<ffffffff8169415d>] thermal_interrupt+0x6d/0x80

Let's defer the work to a kthread.

Signed-off-by Daniel Wagner <[email protected]>
Signed-off-by Steven Rostedt (VMware) <[email protected]>
[bigeasy: reoder init/denit position. TODO: flush swork on exit]
Signed-off-by Sebastian Andrzej Siewior <[email protected]>
---
 drivers/thermal/x86_pkg_temp_thermal.c | 28 +++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)
drivers/thermal/x86_pkg_temp_thermal.c +27 -1
index 1ef937d799e4..82f21fd4afb0 100644
@@ -29,6 +29,7 @@
 #include <linux/pm.h>
 #include <linux/thermal.h>
 #include <linux/debugfs.h>
+#include <linux/kthread.h>
 #include <asm/cpu_device_id.h>
 #include <asm/mce.h>
 

@@ -329,7 +330,7 @@ static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
 	schedule_delayed_work_on(cpu, work, ms);
 }
 
-static int pkg_thermal_notify(u64 msr_val)
+static void pkg_thermal_notify_work(struct kthread_work *work)
 {
 	int cpu = smp_processor_id();
 	struct pkg_device *pkgdev;

@@ -348,8 +349,32 @@ static int pkg_thermal_notify(u64 msr_val)
 	}
 
 	spin_unlock_irqrestore(&pkg_temp_lock, flags);
+}
+
+#ifdef CONFIG_PREEMPT_RT_FULL
+static DEFINE_KTHREAD_WORK(notify_work, pkg_thermal_notify_work);
+
+static int pkg_thermal_notify(u64 msr_val)
+{
+	kthread_schedule_work(&notify_work);
+	return 0;
+}
+
+static void pkg_thermal_notify_flush(void)
+{
+	kthread_flush_work(&notify_work);
+}
+
+#else  /* !CONFIG_PREEMPT_RT_FULL */
+
+static void pkg_thermal_notify_flush(void) { }
+
+static int pkg_thermal_notify(u64 msr_val)
+{
+	pkg_thermal_notify_work(NULL);
 	return 0;
 }
+#endif /* CONFIG_PREEMPT_RT_FULL */
 
 static int pkg_temp_thermal_device_add(unsigned int cpu)
 {

@@ -548,6 +573,7 @@ static void __exit pkg_temp_thermal_exit(void)
 	platform_thermal_package_rate_control = NULL;
 
 	cpuhp_remove_state(pkg_thermal_hp_state);
+	pkg_thermal_notify_flush();
 	debugfs_remove_recursive(debugfs);
 	kfree(packages);
 }
Signature · 1 line
2.20.1
author reviewer
#5 lore raw

RFC PATCH 5/7 revert-block

In reply to Steven Rostedt

From: "Steven Rostedt (VMware)" <[email protected]>

Revert swork version of: block: blk-mq: move blk_queue_usage_counter_release() into process context

In order to switch to upstream, we need to revert the swork code.

Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 block/blk-core.c       | 14 +-------------
 include/linux/blkdev.h |  2 --
 2 files changed, 1 insertion(+), 15 deletions(-)
block/blk-core.c +1 -13
index f7e16b4466f0..3f08d6fd0787 100644
@@ -969,21 +969,12 @@ void blk_queue_exit(struct request_queue *q)
 	percpu_ref_put(&q->q_usage_counter);
 }
 
-static void blk_queue_usage_counter_release_swork(struct swork_event *sev)
-{
-	struct request_queue *q =
-		container_of(sev, struct request_queue, mq_pcpu_wake);
-
-	wake_up_all(&q->mq_freeze_wq);
-}
-
 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
 {
 	struct request_queue *q =
 		container_of(ref, struct request_queue, q_usage_counter);
 
-	if (wq_has_sleeper(&q->mq_freeze_wq))
-		swork_queue(&q->mq_pcpu_wake);
+	wake_up_all(&q->mq_freeze_wq);
 }
 
 static void blk_rq_timed_out_timer(struct timer_list *t)

@@ -1080,7 +1071,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
 
 	init_waitqueue_head(&q->mq_freeze_wq);
-	INIT_SWORK(&q->mq_pcpu_wake, blk_queue_usage_counter_release_swork);
 
 	/*
 	 * Init percpu_ref in atomic mode so that it's faster to shutdown.

@@ -3970,8 +3960,6 @@ int __init blk_dev_init(void)
 	if (!kblockd_workqueue)
 		panic("Failed to create kblockd\n");
 
-	BUG_ON(swork_get());
-
 	request_cachep = kmem_cache_create("blkdev_requests",
 			sizeof(struct request), 0, SLAB_PANIC, NULL);
include/linux/blkdev.h +0 -2
index 7b7c0bc6a514..f1960add94df 100644
@@ -27,7 +27,6 @@
 #include <linux/percpu-refcount.h>
 #include <linux/scatterlist.h>
 #include <linux/blkzoned.h>
-#include <linux/swork.h>
 
 struct module;
 struct scsi_ioctl_command;

@@ -656,7 +655,6 @@ struct request_queue {
 #endif
 	struct rcu_head		rcu_head;
 	wait_queue_head_t	mq_freeze_wq;
-	struct swork_event	mq_pcpu_wake;
 	struct percpu_ref	q_usage_counter;
 	struct list_head	all_q_node;
Signature · 1 line
2.20.1
author reviewer
#6 lore raw

RFC PATCH 6/7 block: blk-mq: move blk_queue_usage_counter_release() into process context

In reply to Steven Rostedt

From: Sebastian Andrzej Siewior <[email protected]>

[ Upstream commit 61c928ecf4fe200bda9b49a0813b5ba0f43995b5 ]

| BUG: sleeping function called from invalid context at kernel/locking/rtmutex.c:914
| in_atomic(): 1, irqs_disabled(): 0, pid: 255, name: kworker/u257:6
| 5 locks held by kworker/u257:6/255:
|  #0:  ("events_unbound"){.+.+.+}, at: [<ffffffff8108edf1>] process_one_work+0x171/0x5e0
|  #1:  ((&entry->work)){+.+.+.}, at: [<ffffffff8108edf1>] process_one_work+0x171/0x5e0
|  #2:  (&shost->scan_mutex){+.+.+.}, at: [<ffffffffa000faa3>] __scsi_add_device+0xa3/0x130 [scsi_mod]
|  #3:  (&set->tag_list_lock){+.+...}, at: [<ffffffff812f09fa>] blk_mq_init_queue+0x96a/0xa50
|  #4:  (rcu_read_lock_sched){......}, at: [<ffffffff8132887d>] percpu_ref_kill_and_confirm+0x1d/0x120
| Preemption disabled at:[<ffffffff812eff76>] blk_mq_freeze_queue_start+0x56/0x70
|
| CPU: 2 PID: 255 Comm: kworker/u257:6 Not tainted 3.18.7-rt0+ #1
| Workqueue: events_unbound async_run_entry_fn
|  0000000000000003 ffff8800bc29f998 ffffffff815b3a12 0000000000000000
|  0000000000000000 ffff8800bc29f9b8 ffffffff8109aa16 ffff8800bc29fa28
|  ffff8800bc5d1bc8 ffff8800bc29f9e8 ffffffff815b8dd4 ffff880000000000
| Call Trace:
|  [<ffffffff815b3a12>] dump_stack+0x4f/0x7c
|  [<ffffffff8109aa16>] __might_sleep+0x116/0x190
|  [<ffffffff815b8dd4>] rt_spin_lock+0x24/0x60
|  [<ffffffff810b6089>] __wake_up+0x29/0x60
|  [<ffffffff812ee06e>] blk_mq_usage_counter_release+0x1e/0x20
|  [<ffffffff81328966>] percpu_ref_kill_and_confirm+0x106/0x120
|  [<ffffffff812eff76>] blk_mq_freeze_queue_start+0x56/0x70
|  [<ffffffff812f0000>] blk_mq_update_tag_set_depth+0x40/0xd0
|  [<ffffffff812f0a1c>] blk_mq_init_queue+0x98c/0xa50
|  [<ffffffffa000dcf0>] scsi_mq_alloc_queue+0x20/0x60 [scsi_mod]
|  [<ffffffffa000ea35>] scsi_alloc_sdev+0x2f5/0x370 [scsi_mod]
|  [<ffffffffa000f494>] scsi_probe_and_add_lun+0x9e4/0xdd0 [scsi_mod]
|  [<ffffffffa000fb26>] __scsi_add_device+0x126/0x130 [scsi_mod]
|  [<ffffffffa013033f>] ata_scsi_scan_host+0xaf/0x200 [libata]
|  [<ffffffffa012b5b6>] async_port_probe+0x46/0x60 [libata]
|  [<ffffffff810978fb>] async_run_entry_fn+0x3b/0xf0
|  [<ffffffff8108ee81>] process_one_work+0x201/0x5e0

percpu_ref_kill_and_confirm() invokes blk_mq_usage_counter_release() in a rcu-sched region. swait based wake queue can't be used due to wake_up_all() usage and disabled interrupts in !RT configs (as reported by Corey Minyard).
The wq_has_sleeper() check has been suggested by Peter Zijlstra.

Signed-off-by Sebastian Andrzej Siewior <[email protected]>
Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 block/blk-core.c       | 12 +++++++++++-
 include/linux/blkdev.h |  2 ++
 2 files changed, 13 insertions(+), 1 deletion(-)
block/blk-core.c +11 -1
index 3f08d6fd0787..8d4c5d69c5c4 100644
@@ -969,12 +969,21 @@ void blk_queue_exit(struct request_queue *q)
 	percpu_ref_put(&q->q_usage_counter);
 }
 
+static void blk_queue_usage_counter_release_wrk(struct kthread_work *work)
+{
+	struct request_queue *q =
+		container_of(work, struct request_queue, mq_pcpu_wake);
+
+	wake_up_all(&q->mq_freeze_wq);
+}
+
 static void blk_queue_usage_counter_release(struct percpu_ref *ref)
 {
 	struct request_queue *q =
 		container_of(ref, struct request_queue, q_usage_counter);
 
-	wake_up_all(&q->mq_freeze_wq);
+	if (wq_has_sleeper(&q->mq_freeze_wq))
+		kthread_schedule_work(&q->mq_pcpu_wake);
 }
 
 static void blk_rq_timed_out_timer(struct timer_list *t)

@@ -1071,6 +1080,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
 
 	init_waitqueue_head(&q->mq_freeze_wq);
+	kthread_init_work(&q->mq_pcpu_wake, blk_queue_usage_counter_release_wrk);
 
 	/*
 	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
include/linux/blkdev.h +2 -0
index f1960add94df..15a489abfb62 100644
@@ -13,6 +13,7 @@
 #include <linux/llist.h>
 #include <linux/timer.h>
 #include <linux/workqueue.h>
+#include <linux/kthread.h>
 #include <linux/pagemap.h>
 #include <linux/backing-dev-defs.h>
 #include <linux/wait.h>

@@ -655,6 +656,7 @@ struct request_queue {
 #endif
 	struct rcu_head		rcu_head;
 	wait_queue_head_t	mq_freeze_wq;
+	struct kthread_work	mq_pcpu_wake;
 	struct percpu_ref	q_usage_counter;
 	struct list_head	all_q_node;
Signature · 1 line
2.20.1
author reviewer
#7 lore raw

RFC PATCH 7/7 workqueue: rework

In reply to Steven Rostedt

From: Sebastian Andrzej Siewior <[email protected]>

[ Upstream commit d15a862f24df983458533aebd6fa207ecdd1095a ]

This is an all-in change of the workqueue rework.
The worker_pool.lock is made to raw_spinlock_t. With this change we can schedule workitems from preempt-disable sections and sections with disabled interrupts. This change allows to remove all kthread_.* workarounds we used to have.

Signed-off-by Sebastian Andrzej Siewior <[email protected]>
Signed-off-by Steven Rostedt (VMware) <[email protected]>
---
 block/blk-core.c                       |   6 +-
 drivers/block/loop.c                   |   2 +-
 drivers/spi/spi-rockchip.c             |   1 -
 drivers/thermal/x86_pkg_temp_thermal.c |  28 +--
 fs/aio.c                               |  10 +-
 include/linux/blk-cgroup.h             |   2 +-
 include/linux/blkdev.h                 |   2 +-
 include/linux/interrupt.h              |   5 -
 include/linux/kthread-cgroup.h         |  17 --
 include/linux/kthread.h                |  15 +-
 include/linux/swait.h                  |  14 ++
 include/linux/workqueue.h              |   4 -
 init/main.c                            |   1 -
 kernel/irq/manage.c                    |  36 +--
 kernel/kthread.c                       |  14 --
 kernel/sched/core.c                    |   1 +
 kernel/time/hrtimer.c                  |  24 --
 kernel/workqueue.c                     | 300 +++++++++++--------------
 18 files changed, 164 insertions(+), 318 deletions(-)
 delete mode 100644 include/linux/kthread-cgroup.h
block/blk-core.c +3 -3
index 8d4c5d69c5c4..f0cb86ef8215 100644
@@ -969,7 +969,7 @@ void blk_queue_exit(struct request_queue *q)
 	percpu_ref_put(&q->q_usage_counter);
 }
 
-static void blk_queue_usage_counter_release_wrk(struct kthread_work *work)
+static void blk_queue_usage_counter_release_wrk(struct work_struct *work)
 {
 	struct request_queue *q =
 		container_of(work, struct request_queue, mq_pcpu_wake);

@@ -983,7 +983,7 @@ static void blk_queue_usage_counter_release(struct percpu_ref *ref)
 		container_of(ref, struct request_queue, q_usage_counter);
 
 	if (wq_has_sleeper(&q->mq_freeze_wq))
-		kthread_schedule_work(&q->mq_pcpu_wake);
+		schedule_work(&q->mq_pcpu_wake);
 }
 
 static void blk_rq_timed_out_timer(struct timer_list *t)

@@ -1080,7 +1080,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
 	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
 
 	init_waitqueue_head(&q->mq_freeze_wq);
-	kthread_init_work(&q->mq_pcpu_wake, blk_queue_usage_counter_release_wrk);
+	INIT_WORK(&q->mq_pcpu_wake, blk_queue_usage_counter_release_wrk);
 
 	/*
 	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
drivers/block/loop.c +1 -1
index 298c96b6befa..cef8e00c9d9d 100644
@@ -70,7 +70,7 @@
 #include <linux/writeback.h>
 #include <linux/completion.h>
 #include <linux/highmem.h>
-#include <linux/kthread-cgroup.h>
+#include <linux/kthread.h>
 #include <linux/splice.h>
 #include <linux/sysfs.h>
 #include <linux/miscdevice.h>
drivers/spi/spi-rockchip.c +0 -1
index b56619418cea..fdcf3076681b 100644
@@ -22,7 +22,6 @@
 #include <linux/spi/spi.h>
 #include <linux/pm_runtime.h>
 #include <linux/scatterlist.h>
-#include <linux/interrupt.h>
 
 #define DRIVER_NAME "rockchip-spi"
drivers/thermal/x86_pkg_temp_thermal.c +1 -27
index 82f21fd4afb0..1ef937d799e4 100644
@@ -29,7 +29,6 @@
 #include <linux/pm.h>
 #include <linux/thermal.h>
 #include <linux/debugfs.h>
-#include <linux/kthread.h>
 #include <asm/cpu_device_id.h>
 #include <asm/mce.h>
 

@@ -330,7 +329,7 @@ static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
 	schedule_delayed_work_on(cpu, work, ms);
 }
 
-static void pkg_thermal_notify_work(struct kthread_work *work)
+static int pkg_thermal_notify(u64 msr_val)
 {
 	int cpu = smp_processor_id();
 	struct pkg_device *pkgdev;

@@ -349,32 +348,8 @@ static void pkg_thermal_notify_work(struct kthread_work *work)
 	}
 
 	spin_unlock_irqrestore(&pkg_temp_lock, flags);
-}
-
-#ifdef CONFIG_PREEMPT_RT_FULL
-static DEFINE_KTHREAD_WORK(notify_work, pkg_thermal_notify_work);
-
-static int pkg_thermal_notify(u64 msr_val)
-{
-	kthread_schedule_work(&notify_work);
-	return 0;
-}
-
-static void pkg_thermal_notify_flush(void)
-{
-	kthread_flush_work(&notify_work);
-}
-
-#else  /* !CONFIG_PREEMPT_RT_FULL */
-
-static void pkg_thermal_notify_flush(void) { }
-
-static int pkg_thermal_notify(u64 msr_val)
-{
-	pkg_thermal_notify_work(NULL);
 	return 0;
 }
-#endif /* CONFIG_PREEMPT_RT_FULL */
 
 static int pkg_temp_thermal_device_add(unsigned int cpu)
 {

@@ -573,7 +548,6 @@ static void __exit pkg_temp_thermal_exit(void)
 	platform_thermal_package_rate_control = NULL;
 
 	cpuhp_remove_state(pkg_thermal_hp_state);
-	pkg_thermal_notify_flush();
 	debugfs_remove_recursive(debugfs);
 	kfree(packages);
 }
fs/aio.c +5 -5
index 0c613d805bf1..c74dd321f5b7 100644
@@ -121,7 +121,7 @@ struct kioctx {
 	long			nr_pages;
 
 	struct rcu_work		free_rwork;	/* see free_ioctx() */
-	struct kthread_work	free_kwork;	/* see free_ioctx() */
+	struct work_struct	free_work;	/* see free_ioctx() */
 
 	/*
 	 * signals when all in-flight requests are done

@@ -607,9 +607,9 @@ static void free_ioctx_reqs(struct percpu_ref *ref)
  * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
  * now it's safe to cancel any that need to be.
  */
-static void free_ioctx_users_work(struct kthread_work *work)
+static void free_ioctx_users_work(struct work_struct *work)
 {
-	struct kioctx *ctx = container_of(work, struct kioctx, free_kwork);
+	struct kioctx *ctx = container_of(work, struct kioctx, free_work);
 	struct aio_kiocb *req;
 
 	spin_lock_irq(&ctx->ctx_lock);

@@ -631,8 +631,8 @@ static void free_ioctx_users(struct percpu_ref *ref)
 {
 	struct kioctx *ctx = container_of(ref, struct kioctx, users);
 
-	kthread_init_work(&ctx->free_kwork, free_ioctx_users_work);
-	kthread_schedule_work(&ctx->free_kwork);
+	INIT_WORK(&ctx->free_work, free_ioctx_users_work);
+	schedule_work(&ctx->free_work);
 }
 
 static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
include/linux/blk-cgroup.h +1 -1
index 0473efda4c65..da587e60fe86 100644
@@ -14,7 +14,7 @@
  * 	              Nauman Rafique <[email protected]>
  */
 
-#include <linux/kthread-cgroup.h>
+#include <linux/kthread.h>
 #include <linux/percpu_counter.h>
 #include <linux/seq_file.h>
 #include <linux/radix-tree.h>
include/linux/blkdev.h +1 -1
index 15a489abfb62..45653e23e4cd 100644
@@ -656,7 +656,7 @@ struct request_queue {
 #endif
 	struct rcu_head		rcu_head;
 	wait_queue_head_t	mq_freeze_wq;
-	struct kthread_work	mq_pcpu_wake;
+	struct work_struct	mq_pcpu_wake;
 	struct percpu_ref	q_usage_counter;
 	struct list_head	all_q_node;
include/linux/interrupt.h +0 -5
index a9321f6429f2..97d9ba26915e 100644
@@ -13,7 +13,6 @@
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
-#include <linux/kthread.h>
 
 #include <linux/atomic.h>
 #include <asm/ptrace.h>

@@ -239,11 +238,7 @@ extern void resume_device_irqs(void);
 struct irq_affinity_notify {
 	unsigned int irq;
 	struct kref kref;
-#ifdef CONFIG_PREEMPT_RT_BASE
-	struct kthread_work work;
-#else
 	struct work_struct work;
-#endif
 	void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
 	void (*release)(struct kref *ref);
 };
include/linux/kthread-cgroup.h deleted +0 -17
deleted file mode 100644
index 53d34bca9d72..000000000000
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_KTHREAD_CGROUP_H
-#define _LINUX_KTHREAD_CGROUP_H
-#include <linux/kthread.h>
-#include <linux/cgroup.h>
-
-#ifdef CONFIG_BLK_CGROUP
-void kthread_associate_blkcg(struct cgroup_subsys_state *css);
-struct cgroup_subsys_state *kthread_blkcg(void);
-#else
-static inline void kthread_associate_blkcg(struct cgroup_subsys_state *css) { }
-static inline struct cgroup_subsys_state *kthread_blkcg(void)
-{
-	return NULL;
-}
-#endif
-#endif
include/linux/kthread.h +9 -6
index 7cf56eb54103..6b8c064f0cbc 100644
@@ -4,6 +4,7 @@
 /* Simple interface for creating and stopping kernel threads without mess. */
 #include <linux/err.h>
 #include <linux/sched.h>
+#include <linux/cgroup.h>
 
 __printf(4, 5)
 struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),

@@ -197,12 +198,14 @@ bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *work);
 
 void kthread_destroy_worker(struct kthread_worker *worker);
 
-extern struct kthread_worker kthread_global_worker;
-void kthread_init_global_worker(void);
-
-static inline bool kthread_schedule_work(struct kthread_work *work)
+#ifdef CONFIG_BLK_CGROUP
+void kthread_associate_blkcg(struct cgroup_subsys_state *css);
+struct cgroup_subsys_state *kthread_blkcg(void);
+#else
+static inline void kthread_associate_blkcg(struct cgroup_subsys_state *css) { }
+static inline struct cgroup_subsys_state *kthread_blkcg(void)
 {
-	return kthread_queue_work(&kthread_global_worker, work);
+	return NULL;
 }
-
+#endif
 #endif /* _LINUX_KTHREAD_H */
include/linux/swait.h +14 -0
index f426a0661aa0..21ae66cd41d3 100644
@@ -299,4 +299,18 @@ do {									\
 	__ret;								\
 })
 
+#define __swait_event_lock_irq(wq, condition, lock, cmd)		\
+	___swait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0,		\
+		       raw_spin_unlock_irq(&lock);			\
+		       cmd;						\
+		       schedule();					\
+		       raw_spin_lock_irq(&lock))
+
+#define swait_event_lock_irq(wq_head, condition, lock)			\
+	do {								\
+		if (condition)						\
+			break;						\
+		__swait_event_lock_irq(wq_head, condition, lock, );	\
+	} while (0)
+
 #endif /* _LINUX_SWAIT_H */
include/linux/workqueue.h +0 -4
index 60d673e15632..546aa73fba6a 100644
@@ -455,10 +455,6 @@ __alloc_workqueue_key(const char *fmt, unsigned int flags, int max_active,
 
 extern void destroy_workqueue(struct workqueue_struct *wq);
 
-struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask);
-void free_workqueue_attrs(struct workqueue_attrs *attrs);
-int apply_workqueue_attrs(struct workqueue_struct *wq,
-			  const struct workqueue_attrs *attrs);
 int workqueue_set_unbound_cpumask(cpumask_var_t cpumask);
 
 extern bool queue_work_on(int cpu, struct workqueue_struct *wq,
init/main.c +0 -1
index b0e95351c22c..4a7471606e53 100644
@@ -1130,7 +1130,6 @@ static noinline void __init kernel_init_freeable(void)
 	smp_prepare_cpus(setup_max_cpus);
 
 	workqueue_init();
-	kthread_init_global_worker();
 
 	init_mm_internals();
kernel/irq/manage.c +3 -33
index 290cd520dba1..82b3728685ca 100644
@@ -285,12 +285,7 @@ int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
 
 	if (desc->affinity_notify) {
 		kref_get(&desc->affinity_notify->kref);
-
-#ifdef CONFIG_PREEMPT_RT_BASE
-		kthread_schedule_work(&desc->affinity_notify->work);
-#else
 		schedule_work(&desc->affinity_notify->work);
-#endif
 	}
 	irqd_set(data, IRQD_AFFINITY_SET);
 

@@ -328,8 +323,10 @@ int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
 }
 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
 
-static void _irq_affinity_notify(struct irq_affinity_notify *notify)
+static void irq_affinity_notify(struct work_struct *work)
 {
+	struct irq_affinity_notify *notify =
+		container_of(work, struct irq_affinity_notify, work);
 	struct irq_desc *desc = irq_to_desc(notify->irq);
 	cpumask_var_t cpumask;
 	unsigned long flags;

@@ -351,25 +348,6 @@ static void _irq_affinity_notify(struct irq_affinity_notify *notify)
 	kref_put(&notify->kref, notify->release);
 }
 
-#ifdef CONFIG_PREEMPT_RT_BASE
-
-static void irq_affinity_notify(struct kthread_work *work)
-{
-	struct irq_affinity_notify *notify =
-		container_of(work, struct irq_affinity_notify, work);
-	_irq_affinity_notify(notify);
-}
-
-#else
-
-static void irq_affinity_notify(struct work_struct *work)
-{
-	struct irq_affinity_notify *notify =
-		container_of(work, struct irq_affinity_notify, work);
-	_irq_affinity_notify(notify);
-}
-#endif
-
 /**
  *	irq_set_affinity_notifier - control notification of IRQ affinity changes
  *	@irq:		Interrupt for which to enable/disable notification

@@ -398,11 +376,7 @@ irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 	if (notify) {
 		notify->irq = irq;
 		kref_init(&notify->kref);
-#ifdef CONFIG_PREEMPT_RT_BASE
-		kthread_init_work(&notify->work, irq_affinity_notify);
-#else
 		INIT_WORK(&notify->work, irq_affinity_notify);
-#endif
 	}
 
 	raw_spin_lock_irqsave(&desc->lock, flags);

@@ -411,11 +385,7 @@ irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 
 	if (old_notify) {
-#ifdef CONFIG_PREEMPT_RT_BASE
-		kthread_cancel_work_sync(&notify->work);
-#else
 		cancel_work_sync(&old_notify->work);
-#endif
 		kref_put(&old_notify->kref, old_notify->release);
 	}
kernel/kthread.c +0 -14
index 9db017761a1f..5641b55783a6 100644
@@ -20,7 +20,6 @@
 #include <linux/freezer.h>
 #include <linux/ptrace.h>
 #include <linux/uaccess.h>
-#include <linux/cgroup.h>
 #include <trace/events/sched.h>
 
 static DEFINE_SPINLOCK(kthread_create_lock);

@@ -1181,19 +1180,6 @@ void kthread_destroy_worker(struct kthread_worker *worker)
 }
 EXPORT_SYMBOL(kthread_destroy_worker);
 
-DEFINE_KTHREAD_WORKER(kthread_global_worker);
-EXPORT_SYMBOL(kthread_global_worker);
-
-__init void kthread_init_global_worker(void)
-{
-	kthread_global_worker.task = kthread_create(kthread_worker_fn,
-						    &kthread_global_worker,
-						    "kswork");
-	if (WARN_ON(IS_ERR(kthread_global_worker.task)))
-		return;
-	wake_up_process(kthread_global_worker.task);
-}
-
 #ifdef CONFIG_BLK_CGROUP
 /**
  * kthread_associate_blkcg - associate blkcg to current kthread
kernel/sched/core.c +1 -0
index 34e786c60d79..dd14d3960ab8 100644
@@ -3566,6 +3566,7 @@ static inline void sched_submit_work(struct task_struct *tsk)
 {
 	if (!tsk->state)
 		return;
+
 	/*
 	 * If a worker went to sleep, notify and ask workqueue whether
 	 * it wants to wake up a task to maintain concurrency.
kernel/time/hrtimer.c +0 -24
index 4534e7871c8c..67951292df58 100644
@@ -730,29 +730,6 @@ static void hrtimer_switch_to_hres(void)
 	retrigger_next_event(NULL);
 }
 
-#ifdef CONFIG_PREEMPT_RT_FULL
-
-static struct swork_event clock_set_delay_work;
-
-static void run_clock_set_delay(struct swork_event *event)
-{
-	clock_was_set();
-}
-
-void clock_was_set_delayed(void)
-{
-	swork_queue(&clock_set_delay_work);
-}
-
-static __init int create_clock_set_delay_thread(void)
-{
-	WARN_ON(swork_get());
-	INIT_SWORK(&clock_set_delay_work, run_clock_set_delay);
-	return 0;
-}
-early_initcall(create_clock_set_delay_thread);
-#else /* PREEMPT_RT_FULL */
-
 static void clock_was_set_work(struct work_struct *work)
 {
 	clock_was_set();

@@ -768,7 +745,6 @@ void clock_was_set_delayed(void)
 {
 	schedule_work(&hrtimer_work);
 }
-#endif
 
 #else
kernel/workqueue.c +125 -175

940 lines not on this page · raw message

Signature · 1 line
2.20.1