mfd/rk8xx: fix sleep during RCU read side critical section
PATCH 1/1 mfd/rk8xx: fix sleep during RCU read side critical section
SPI does not support atomic transfers, so it's dangerous to call from atomic notif chains, since it can cause stall during reboot on orangepi5 (see trace below).
Commit 128115b520c6 ("mfd: rk8xx: Register poweroff handler in atomic phase") moved shutdown methods to atomic notif chain, since i2c transfer didn't work during PREPARE step. I don't have an access to rk3288 based boards and patch reasoning is not completely clear.
That's why this patch hacks notifier registration only for SPI based PMICs.
66.145226] reboot: Power down
[ 266.145581] ------------[ cut here ]------------ [ 266.145986] Voluntary context switch within RCU read-side critical section! [ 266.146007] WARNING: kernel/rcu/tree_plugin.h:330 at rcu_note_context_switch+0x3dc/0x5d8, CPU#0: systemd-shutdow/1 [ 266.147525] Modules linked in: [ 266.147800] CPU: 0 UID: 0 PID: 1 Comm: systemd-shutdow Not tainted 7.3.0-opi5-test+ #2 PREEMPT [ 266.148560] Hardware name: Xunlong Orange Pi 5 (DT) [ 266.148987] pstate: 604000c9 (nZCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 266.149596] pc : rcu_note_context_switch+0x3dc/0x5d8 [ 266.150034] lr : rcu_note_context_switch+0x3dc/0x5d8 [ 266.150472] sp : ffff80008005b4e0 [ 266.150764] x29: ffff80008005b4e0 x28: ffff000100903000 x27: ffffc7018d616000 [ 266.151393] x26: 0000c4fe4deb68c0 x25: ffffc7018d639260 x24: 0000000000000000 [ 266.152021] x23: ffff000100903000 x22: ffff0002fee4ef80 x21: ffffc7018d616f80 [ 266.152649] x20: 0000000000000000 x19: ffff000100903000 x18: 0000000000000000 [ 266.153277] x17: ffff80008005b8b8 x16: 0000000000000000 x15: 0000000000000000 [ 266.153904] x14: 0000000000000a37 x13: ffffc7018d82a478 x12: ffff390171838000 [ 266.154532] x11: ffff0002fee3a198 x10: 0000000000000001 x9 : ffffc7018d6642d0 [ 266.155161] x8 : 0000000100000001 x7 : 0000000000000000 x6 : 7261746e756c6f56 [ 266.155789] x5 : ffffc7018d93e807 x4 : ffffc7018cd48369 x3 : ffff80008005b058 [ 266.156416] x2 : 00000000000002a8 x1 : 0000000000000001 x0 : 000000000000003f [ 266.157044] Call trace: [ 266.157262] rcu_note_context_switch+0x3dc/0x5d8 (P) [ 266.157703] __schedule+0x80/0x1540 [ 266.158015] schedule+0x5c/0xe0 [ 266.158296] schedule_timeout+0x7c/0xe8 [ 266.158636] wait_for_common+0xbc/0x188 [ 266.158977] wait_for_completion_timeout+0x1c/0x30 [ 266.159401] spi_transfer_one_message+0x2c0/0x838 [ 266.159819] __spi_pump_transfer_message+0x1f0/0x448 [ 266.160258] __spi_sync+0x224/0x450 [ 266.160569] spi_write_then_read+0x134/0x1b8 [ 266.160947] rk806_spi_bus_read+0x4c/0x70 [ 266.161304] _regmap_raw_read+0x1a0/0x368 [ 266.161661] _regmap_bus_read+0x48/0x88 [ 266.162001] _regmap_read+0x17c/0x260 [ 266.162326] _regmap_update_bits+0x7c/0x138 [ 266.162696] regmap_update_bits_base+0x6c/0xa8 [ 266.163089] rk808_power_off+0xec/0x120 [ 266.163429] sys_off_notify+0x34/0x50 [ 266.163756] atomic_notifier_call_chain+0x78/0x170 [ 266.164178] do_kernel_power_off+0xf0/0x170 [ 266.164549] machine_power_off+0x20/0x38 [ 266.164900] kernel_power_off+0xf4/0x110 [ 266.165248] __arm64_sys_reboot+0x27c/0x290 [ 266.165619] invoke_syscall+0x4c/0xf8 [ 266.165944] el0_svc_common+0x94/0xe8 [ 266.166268] do_el0_svc+0x24/0x38 [ 266.166562] el0_svc+0x34/0x90 [ 266.166833] el0t_64_sync_handler+0x68/0xe0 [ 266.167203] el0t_64_sync+0x17c/0x180 [ 266.167528] ---[ end trace 0000000000000000 ]---
---
Hey Eric!
I don't have rk3288 board to test. However, I strongly suspect that patch does not affect your case.
I've tried to track down root cause of i2c hung, but no luck. Based on FDT i2c is directly connected to GIC, so IRQs must be enabled during PREPARE step.
Would be great if you test that patch.
--- drivers/mfd/rk8xx-core.c | 19 ++++++++++++++++--- drivers/mfd/rk8xx-i2c.c | 2 +- drivers/mfd/rk8xx-spi.c | 2 +- include/linux/mfd/rk808.h | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-)
drivers/mfd/rk8xx-core.c +16 -3
@@ -761,7 +761,7 @@ void rk8xx_shutdown(struct device *dev) }EXPORT_SYMBOL_GPL (rk8xx_shutdown); -int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap) +int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap,bool spi) {struct rk808 *rk808;const struct rk808_reg_data *pre_init_reg;
@@ -874,8 +874,21 @@ int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmapif (device_property_read_bool (dev,"system-power-controller" ) ||device_property_read_bool (dev,"rockchip,system-power-controller" )) { +/* There is no support for atomic SPI transfers (yet?). Thus it's not possible to +* perform SPI i/o during atomic notify handler. +* +* Patch 128115b520c6 claims that i2c does not work during _PREPARE state, so be +* extra careful while changing modes here. +*/ +enum sys_off_mode off_mode = + spi ?SYS_OFF_MODE_POWER_OFF_PREPARE : +SYS_OFF_MODE_POWER_OFF ; +enum sys_off_mode restart_mode = + spi ?SYS_OFF_MODE_RESTART_PREPARE : +SYS_OFF_MODE_RESTART ; + ret =devm_register_sys_off_handler (dev, -SYS_OFF_MODE_POWER_OFF ,SYS_OFF_PRIO_HIGH , + off_mode,SYS_OFF_PRIO_HIGH , &rk808_power_off, rk808);if (ret) return dev_err_probe(dev, ret,
@@ -885,7 +898,7 @@ int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmapcase RK809_ID :case RK817_ID : ret =devm_register_sys_off_handler (dev, -SYS_OFF_MODE_RESTART ,SYS_OFF_PRIO_HIGH , + restart_mode,SYS_OFF_PRIO_HIGH , &rk808_restart, rk808);if (ret)dev_warn (dev,"failed to register rst handler, %d\n" , ret);
drivers/mfd/rk8xx-i2c.c +1 -1
@@ -243,7 +243,7 @@ static int rk8xx_i2c_probe(struct i2c_client *client)return dev_err_probe (&client->dev,PTR_ERR (regmap),"regmap initialization failed\n" ); -return rk8xx_probe (&client->dev, data->variant, client->irq, regmap); +return rk8xx_probe (&client->dev, data->variant, client->irq, regmap, false); }static void rk8xx_i2c_shutdown (struct i2c_client *client)
drivers/mfd/rk8xx-spi.c +1 -1
@@ -94,7 +94,7 @@ static int rk8xx_spi_probe(struct spi_device *spi)return dev_err_probe (&spi->dev,PTR_ERR (regmap),"Failed to init regmap\n" ); -return rk8xx_probe (&spi->dev,RK806_ID , spi->irq, regmap); +return rk8xx_probe (&spi->dev,RK806_ID , spi->irq, regmap, true); }static const struct of_device_id rk8xx_spi_of_match[] = {
include/linux/mfd/rk808.h +1 -1
@@ -1469,7 +1469,7 @@ struct rk808 { };void rk8xx_shutdown (struct device *dev); -int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap); +int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap,bool spi);int rk8xx_suspend (struct device *dev);int rk8xx_resume (struct device *dev);
Signature · 1 line
2.55.0
[email protected]
AI reviewSashiko has reviewed this patch and found no issues.
Quoted 13 lines
SPI does not support atomic transfers, so it's dangerous to call from atomic notif chains, since it can cause stall during reboot on orangepi5 (see trace below).
Commit
128115b520c6("mfd: rk8xx: Register poweroff handler in atomic phase") moved shutdown methods to atomic notif chain, since i2c transfer didn't work during PREPARE step. I don't have an access to rk3288 based boards and patch reasoning is not completely clear.That's why this patch hacks notifier registration only for SPI based PMICs.
66.145226] reboot: Power down
Quoted 8 lines
[ 266.145581] ------------[ cut here ]------------ [ 266.145986] Voluntary context switch within RCU read-side critical section! [ ... ] [ 266.167528] ---[ end trace 0000000000000000 ]---CC Eric Anderson <[email protected]>Fixes 128115b520c6 ("mfd: rk8xx: Register poweroff handler in atomic phase")Signed-off-by Pavel Skripkin <[email protected]>
Sashiko has reviewed this patch and found no issues.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=1
Quoted 3 lines · Pavel Skripkin
SPI does not support atomic transfers, so it's dangerous to call from atomic notif chains, since it can cause stall during reboot on orangepi5 (see trace below).
For my curiosity, is the hang during reboot/shutdown random or consistent?
You are explicitly mentioning reboot, although the stack trace is for shutdown. I'd have expected reboot to have always been broken for you. In my change I noticed that reboot and shutdown didn't agree on whether to use _PREPARE, and my change adjusted shutdown to agree with reboot. Since you apparently need the opposite, prior to my change I'd assume shutdown was working but reboot broken.
Quoted 2 lines
I don't have rk3288 board to test. However, I strongly suspect that patch does not affect your case.
I'll test it this weekend. I suspect it would work fine on Veyron/rk3288.
Hi Eric,
Eric Anderson <[email protected]> says:
Quoted 7 lines · Pavel Skripkin
Quoted 3 lines
SPI does not support atomic transfers, so it's dangerous to call from atomic notif chains, since it can cause stall during reboot on orangepi5 (see trace below).
For my curiosity, is the hang during reboot/shutdown random or consistent?
They are basically the same, since they both trigger sleeping SPI I/O. I agree, I should have used studown/reboot more consistently in commit message...
Quoted 2 lines
You are explicitly mentioning reboot, although the stack trace is for shutdown. I'd have expected reboot to have always been broken for you.
Likely. I've just bought it, so cannot say if it was always broken. Based on code I believe that it was indeed broken from the beginning. Also I've found previous attempt to fix it [1], so looks like problem is known for quite a long time
Quoted 8 lines
In my change I noticed that reboot and shutdown didn't agree on whether to use
_PREPARE, and my change adjusted shutdown to agree with reboot. Since you apparently need the opposite, prior to my change I'd assume shutdown was working but reboot broken.Quoted 1 line
I don't have rk3288 board to test. However, I strongly suspect that>> patch does not affect your case.
I'll test it this weekend. I suspect it would work fine on Veyron/rk3288.
[1] https://lists.infradead.org/pipermail/linux-rockchip/2024-July/049312.html
With regards,
Pavel Skripkin
Pavel,
I tested the patch, and it didn't impact shutdown/restart of my Veyron device (it continued working properly).
Quoted 3 lines · Pavel Skripkin
Based on code I believe that it was indeed broken from the beginning. Also I've found previous attempt to fix it [1], so looks like problem is known for quite a long time
So Sebastian noticed the i2c regression due to his earlier change. But spi wasn't actually broken when using SYS_OFF_MODE_POWER_OFF (as the code is now); the board still powered down, just with the WARNING noise. The thread on lore[1] is easier to follow, as it isn't split into months.
In the thread, the feeling is the approach is a hack, because you don't actually want to poweroff during _PREPARE. And Lee is concerned about the hack:
Quoted 1 line
What's the bet that if accepted, this hack is still here in 5 years time?
The main argument for using _PREPARE for SPI is that it is a small, simple workaround and could go in the LTS. Interestingly, because that patch wasn't accepted, the hack continued living and i2c suffered. My change then removed the spi hack.
Lee Jones
maintainerNit for when you resubmit this.
The subject line does not match the expected format for the subsystem.
<end>
Quoted 175 lines · Pavel Skripkin
SPI does not support atomic transfers, so it's dangerous to call from atomic notif chains, since it can cause stall during reboot on orangepi5 (see trace below).
Commit
128115b520c6("mfd: rk8xx: Register poweroff handler in atomic phase") moved shutdown methods to atomic notif chain, since i2c transfer didn't work during PREPARE step. I don't have an access to rk3288 based boards and patch reasoning is not completely clear.That's why this patch hacks notifier registration only for SPI based PMICs.
66.145226] reboot: Power down
[ 266.145581] ------------[ cut here ]------------ [ 266.145986] Voluntary context switch within RCU read-side critical section! [ 266.146007] WARNING: kernel/rcu/tree_plugin.h:330 at rcu_note_context_switch+0x3dc/0x5d8, CPU#0: systemd-shutdow/1 [ 266.147525] Modules linked in: [ 266.147800] CPU: 0 UID: 0 PID: 1 Comm: systemd-shutdow Not tainted 7.3.0-opi5-test+ #2 PREEMPT [ 266.148560] Hardware name: Xunlong Orange Pi 5 (DT) [ 266.148987] pstate: 604000c9 (nZCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 266.149596] pc : rcu_note_context_switch+0x3dc/0x5d8 [ 266.150034] lr : rcu_note_context_switch+0x3dc/0x5d8 [ 266.150472] sp : ffff80008005b4e0 [ 266.150764] x29: ffff80008005b4e0 x28: ffff000100903000 x27: ffffc7018d616000 [ 266.151393] x26: 0000c4fe4deb68c0 x25: ffffc7018d639260 x24: 0000000000000000 [ 266.152021] x23: ffff000100903000 x22: ffff0002fee4ef80 x21: ffffc7018d616f80 [ 266.152649] x20: 0000000000000000 x19: ffff000100903000 x18: 0000000000000000 [ 266.153277] x17: ffff80008005b8b8 x16: 0000000000000000 x15: 0000000000000000 [ 266.153904] x14: 0000000000000a37 x13: ffffc7018d82a478 x12: ffff390171838000 [ 266.154532] x11: ffff0002fee3a198 x10: 0000000000000001 x9 : ffffc7018d6642d0 [ 266.155161] x8 : 0000000100000001 x7 : 0000000000000000 x6 : 7261746e756c6f56 [ 266.155789] x5 : ffffc7018d93e807 x4 : ffffc7018cd48369 x3 : ffff80008005b058 [ 266.156416] x2 : 00000000000002a8 x1 : 0000000000000001 x0 : 000000000000003f [ 266.157044] Call trace: [ 266.157262] rcu_note_context_switch+0x3dc/0x5d8 (P) [ 266.157703] __schedule+0x80/0x1540 [ 266.158015] schedule+0x5c/0xe0 [ 266.158296] schedule_timeout+0x7c/0xe8 [ 266.158636] wait_for_common+0xbc/0x188 [ 266.158977] wait_for_completion_timeout+0x1c/0x30 [ 266.159401] spi_transfer_one_message+0x2c0/0x838 [ 266.159819] __spi_pump_transfer_message+0x1f0/0x448 [ 266.160258] __spi_sync+0x224/0x450 [ 266.160569] spi_write_then_read+0x134/0x1b8 [ 266.160947] rk806_spi_bus_read+0x4c/0x70 [ 266.161304] _regmap_raw_read+0x1a0/0x368 [ 266.161661] _regmap_bus_read+0x48/0x88 [ 266.162001] _regmap_read+0x17c/0x260 [ 266.162326] _regmap_update_bits+0x7c/0x138 [ 266.162696] regmap_update_bits_base+0x6c/0xa8 [ 266.163089] rk808_power_off+0xec/0x120 [ 266.163429] sys_off_notify+0x34/0x50 [ 266.163756] atomic_notifier_call_chain+0x78/0x170 [ 266.164178] do_kernel_power_off+0xf0/0x170 [ 266.164549] machine_power_off+0x20/0x38 [ 266.164900] kernel_power_off+0xf4/0x110 [ 266.165248] __arm64_sys_reboot+0x27c/0x290 [ 266.165619] invoke_syscall+0x4c/0xf8 [ 266.165944] el0_svc_common+0x94/0xe8 [ 266.166268] do_el0_svc+0x24/0x38 [ 266.166562] el0_svc+0x34/0x90 [ 266.166833] el0t_64_sync_handler+0x68/0xe0 [ 266.167203] el0t_64_sync+0x17c/0x180 [ 266.167528] ---[ end trace 0000000000000000 ]---CC Eric Anderson <[email protected]>Fixes 128115b520c6 ("mfd: rk8xx: Register poweroff handler in atomic phase")Signed-off-by Pavel Skripkin <[email protected]>---
Hey Eric!
I don't have rk3288 board to test. However, I strongly suspect that patch does not affect your case.
I've tried to track down root cause of i2c hung, but no luck. Based on FDT i2c is directly connected to GIC, so IRQs must be enabled during PREPARE step.
Would be great if you test that patch.
--- drivers/mfd/rk8xx-core.c | 19 ++++++++++++++++--- drivers/mfd/rk8xx-i2c.c | 2 +- drivers/mfd/rk8xx-spi.c | 2 +- include/linux/mfd/rk808.h | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-)drivers/mfd/rk8xx-core.c +16
-3@@ -761,7 +761,7 @@ void rk8xx_shutdown(struct device *dev) }EXPORT_SYMBOL_GPL (rk8xx_shutdown); -int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap) +int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap,bool spi) {struct rk808 *rk808;const struct rk808_reg_data *pre_init_reg;@@ -874,8 +874,21 @@ int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmapif (device_property_read_bool (dev,"system-power-controller" ) ||device_property_read_bool (dev,"rockchip,system-power-controller" )) { +/* There is no support for atomic SPI transfers (yet?). Thus it's not possible to +* perform SPI i/o during atomic notify handler. +* +* Patch 128115b520c6 claims that i2c does not work during _PREPARE state, so be +* extra careful while changing modes here. +*/ +enum sys_off_mode off_mode = + spi ?SYS_OFF_MODE_POWER_OFF_PREPARE : +SYS_OFF_MODE_POWER_OFF ; +enum sys_off_mode restart_mode = + spi ?SYS_OFF_MODE_RESTART_PREPARE : +SYS_OFF_MODE_RESTART ; + ret =devm_register_sys_off_handler (dev, -SYS_OFF_MODE_POWER_OFF ,SYS_OFF_PRIO_HIGH , + off_mode,SYS_OFF_PRIO_HIGH , &rk808_power_off, rk808);if (ret) return dev_err_probe(dev, ret,@@ -885,7 +898,7 @@ int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmapcase RK809_ID :case RK817_ID : ret =devm_register_sys_off_handler (dev, -SYS_OFF_MODE_RESTART ,SYS_OFF_PRIO_HIGH , + restart_mode,SYS_OFF_PRIO_HIGH , &rk808_restart, rk808);if (ret)dev_warn (dev,"failed to register rst handler, %d\n" , ret);drivers/mfd/rk8xx-i2c.c +1
-1@@ -243,7 +243,7 @@ static int rk8xx_i2c_probe(struct i2c_client *client)return dev_err_probe (&client->dev,PTR_ERR (regmap),"regmap initialization failed\n" ); -return rk8xx_probe (&client->dev, data->variant, client->irq, regmap); +return rk8xx_probe (&client->dev, data->variant, client->irq, regmap, false); }static void rk8xx_i2c_shutdown (struct i2c_client *client)drivers/mfd/rk8xx-spi.c +1
-1@@ -94,7 +94,7 @@ static int rk8xx_spi_probe(struct spi_device *spi)return dev_err_probe (&spi->dev,PTR_ERR (regmap),"Failed to init regmap\n" ); -return rk8xx_probe (&spi->dev,RK806_ID , spi->irq, regmap); +return rk8xx_probe (&spi->dev,RK806_ID , spi->irq, regmap, true); }static const struct of_device_id rk8xx_spi_of_match[] = {include/linux/mfd/rk808.h +1
-1@@ -1469,7 +1469,7 @@ struct rk808 { };void rk8xx_shutdown (struct device *dev); -int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap); +int rk8xx_probe (struct device *dev,int variant,unsigned int irq,struct regmap *regmap,bool spi);int rk8xx_suspend (struct device *dev);int rk8xx_resume (struct device *dev);Signature · 1 line
2.55.0
Signature · 1 line
Lee Jones