PM: domains: Introduce .power_pre/post_on/off callbacks
Subsystems (7)
RFC PATCH 1/3 PM: domains: Introduce .power_pre/post_on/off callbacks
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core prepare_lock second, while another thread could do the reverse, and this would trigger lockdep warning.
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the genpd_lock() and after genpd_unlock() respectively in case the domain is powered on and off. Those are meant to let drivers claim clock core prepare_lock via clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.
---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]
--- drivers/base/power/domain.c | 103 ++++++++++++++++++++++++++++++++---- include/linux/pm_domain.h | 4 ++ 2 files changed, 97 insertions(+), 10 deletions(-)
drivers/base/power/domain.c +93 -10
@@ -494,6 +494,22 @@ void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) }EXPORT_SYMBOL_GPL (dev_pm_genpd_set_next_wakeup); +static int genpd_power_pre_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_pre_on) +return 0 ; + +return genpd->power_pre_on (genpd); +} + +static int genpd_power_post_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_post_on) +return 0 ; + +return genpd->power_post_on (genpd); +} +static int _genpd_power_on (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;
@@ -544,6 +560,22 @@ static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)return ret; } +static int genpd_power_off_pre (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_pre) +return 0 ; + +return genpd->power_off_pre (genpd); +} + +static int genpd_power_off_post (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_post) +return 0 ; + +return genpd->power_off_post (genpd); +} +static int _genpd_power_off (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;
@@ -816,12 +848,18 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,static void genpd_power_off_work_fn (struct work_struct *work) {struct generic_pm_domain *genpd; +int ret; genpd = container_of(work, struct generic_pm_domain, power_off_work); + ret =genpd_power_off_pre (genpd); +if (ret) +return ;genpd_lock (genpd);genpd_power_off (genpd, false,0 );genpd_unlock (genpd); + ret =genpd_power_off_post (genpd); +WARN_ON_ONCE (ret); }/**
@@ -938,12 +976,14 @@ static int genpd_runtime_suspend(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd))return 0 ; + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); gpd_data->rpm_pstate =genpd_drop_performance_state (dev);genpd_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/**
@@ -977,12 +1017,21 @@ static int genpd_runtime_resume(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd)) goto out; + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd); ret =genpd_power_on (genpd,0 );if (!ret)genpd_restore_performance_state (dev, gpd_data->rpm_pstate);genpd_unlock (genpd); +if (ret) { +genpd_power_post_on (genpd); +return ret; + } + + ret =genpd_power_post_on (genpd);if (ret)return ret;
@@ -1017,10 +1066,13 @@ static int genpd_runtime_resume(struct device *dev)genpd_stop_dev (genpd, dev); err_poweroff:if (!pm_runtime_is_irq_safe (dev) ||genpd_is_irq_safe (genpd)) { -genpd_lock (genpd); - gpd_data->rpm_pstate =genpd_drop_performance_state (dev); -genpd_power_off (genpd, true,0 ); -genpd_unlock (genpd); +if (!genpd_power_off_pre (genpd)) { +genpd_lock (genpd); + gpd_data->rpm_pstate =genpd_drop_performance_state (dev); +genpd_power_off (genpd, true,0 ); +genpd_unlock (genpd); +genpd_power_off_post (genpd); + } }return ret;
@@ -1225,12 +1277,14 @@ static int genpd_finish_suspend(struct device *dev, bool poweroff) } } + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); genpd->suspended_count++;genpd_sync_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/**
@@ -1267,10 +1321,16 @@ static int genpd_resume_noirq(struct device *dev)if (device_wakeup_path (dev) &&genpd_is_active_wakeup (genpd))return pm_generic_resume_noirq (dev); + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);genpd_sync_power_on (genpd, true,0 ); genpd->suspended_count--;genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {
@@ -1378,6 +1438,9 @@ static int genpd_restore_noirq(struct device *dev)* At this point suspended_count == 0 means we are being run for the * first time for the given domain in the present cycle. */ + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);if (genpd->suspended_count++ ==0 ) {/*
@@ -1390,6 +1453,9 @@ static int genpd_restore_noirq(struct device *dev)genpd_sync_power_on (genpd, true,0 );genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {
@@ -1413,6 +1479,7 @@ static int genpd_restore_noirq(struct device *dev)static void genpd_complete (struct device *dev) {struct generic_pm_domain *genpd; +int ret;dev_dbg (dev,"%s()\n" , __func__);
@@ -1435,6 +1502,7 @@ static void genpd_switch_state(struct device *dev, bool suspend) {struct generic_pm_domain *genpd;bool use_lock; +int ret; genpd =dev_to_genpd_safe (dev);if (!genpd)
@@ -1442,8 +1510,13 @@ static void genpd_switch_state(struct device *dev, bool suspend) use_lock =genpd_is_irq_safe (genpd); -if (use_lock) +if (use_lock) { + ret = suspend ?genpd_power_off_pre (genpd) : +genpd_power_pre_on (genpd); +if (ret) +return ;genpd_lock (genpd); + }if (suspend) { genpd->suspended_count++;
@@ -1453,8 +1526,12 @@ static void genpd_switch_state(struct device *dev, bool suspend) genpd->suspended_count--; } -if (use_lock) +if (use_lock) {genpd_unlock (genpd); + ret = suspend ?genpd_power_off_post (genpd) : +genpd_power_post_on (genpd); +WARN_ON_ONCE (ret); + } }/**
@@ -2750,9 +2827,15 @@ static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev, dev->pm_domain->sync = genpd_dev_pm_sync;if (power_on) { + ret =genpd_power_pre_on (pd); +if (ret) +return ret;genpd_lock (pd); ret =genpd_power_on (pd,0 );genpd_unlock (pd); + ret =genpd_power_post_on (pd); +if (ret) +return ret; }if (ret) {
include/linux/pm_domain.h +4 -0
@@ -134,8 +134,12 @@ struct generic_pm_domain {unsigned int prepared_count;/* Suspend counter of prepared devices */ unsigned int performance_state;/* Aggregated max performance state */ cpumask_var_t cpus;/* A cpumask of the attached CPUs */ +int (*power_off_pre)(struct generic_pm_domain *domain);int (*power_off)(struct generic_pm_domain *domain); +int (*power_off_post)(struct generic_pm_domain *domain); +int (*power_pre_on)(struct generic_pm_domain *domain);int (*power_on)(struct generic_pm_domain *domain); +int (*power_post_on)(struct generic_pm_domain *domain);struct raw_notifier_head power_notifiers;/* Power on/off notifiers */ struct opp_table *opp_table;/* OPP table of the genpd */ unsigned int (*opp_to_performance_state)(struct generic_pm_domain *genpd,
Signature · 1 line
2.35.1
RFC PATCH 2/3 soc: imx: gpcv2: Split clock prepare from clock enable in the domain
It is possible for clk_disable_unused() to trigger lockdep warning regarding lock ordering in this driver. This happens in case of the following conditions:
A) clock core clk_disable_unused() triggers the following sequence in a driver which also uses GPCv2 domain:
- clk_prepare_lock() -> obtains clock core prepare_lock
- pm_runtime_get*() -> obtains &blk_ctrl_genpd_lock_class
B) driver powers up a power domain and triggers the following sequence in GPCv2:
- pm_runtime_get_sync() -> obtains &blk_ctrl_genpd_lock_class
- clk_bulk_prepare_enable() -> obtains clock core prepare_lock
This can lead to a deadlock in case A and B runs on separate CPUs.
To avoid the deadlock, split clk_*prepare() from clk_*enable() and call the former in power_pre_on() callback, before pm_runtime_get_sync(). The reverse is implemented in the power_off_post() callback in the same way. This way, the GPCv2 driver always claims the prepare_lock before blk_ctrl_genpd_lock_class and the deadlock is avoided.
---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]
--- drivers/soc/imx/gpcv2.c | 74 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 8 deletions(-)
drivers/soc/imx/gpcv2.c +66 -8
@@ -298,6 +298,8 @@ struct imx_pgc_domain {unsigned int pgc_sw_pup_reg;unsigned int pgc_sw_pdn_reg; + +int enabled; };struct imx_pgc_domain_data {
@@ -313,6 +315,52 @@ to_imx_pgc_domain(struct generic_pm_domain *genpd) return container_of(genpd, struct imx_pgc_domain, genpd); } +static int imx_pgc_power_pre_up (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); +int ret; + + ret =clk_bulk_prepare (domain->num_clks, domain->clks); +if (ret) +dev_err (domain->dev,"failed to prepare reset clocks\n" ); + +return ret; +} + +static int imx_pgc_power_post_up (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); + +if (!domain->keep_clocks && domain->enabled) +clk_bulk_unprepare (domain->num_clks, domain->clks); + +return 0 ; +} + +static int imx_pgc_power_down_pre (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); +int ret; + +if (!domain->keep_clocks || !domain->enabled) { + ret =clk_bulk_prepare (domain->num_clks, domain->clks); +if (ret) +dev_err (domain->dev,"failed to prepare reset clocks\n" ); + } + +return ret; +} + +static int imx_pgc_power_down_post (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); + +if (!domain->keep_clocks || !domain->enabled) +clk_bulk_unprepare (domain->num_clks, domain->clks); + +return 0 ; +} +static int imx_pgc_power_up (struct generic_pm_domain *genpd) {struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd);
@@ -338,7 +386,7 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)reset_control_assert (domain->reset);/* Enable reset clocks for all devices in the domain */ - ret =clk_bulk_prepare_enable (domain->num_clks, domain->clks); + ret =clk_bulk_enable (domain->num_clks, domain->clks);if (ret) {dev_err (domain->dev,"failed to enable reset clocks\n" ); goto out_regulator_disable;
@@ -397,12 +445,14 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)/* Disable reset clocks for all devices in the domain */ if (!domain->keep_clocks) -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); + + domain->enabled++;return 0 ; out_clk_disable: -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); out_regulator_disable:if (!IS_ERR (domain->regulator))regulator_disable (domain->regulator);
@@ -420,7 +470,7 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd)/* Enable reset clocks for all devices in the domain */ if (!domain->keep_clocks) { - ret =clk_bulk_prepare_enable (domain->num_clks, domain->clks); + ret =clk_bulk_enable (domain->num_clks, domain->clks);if (ret) {dev_err (domain->dev,"failed to enable reset clocks\n" );return ret;
@@ -467,7 +517,7 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd) }/* Disable reset clocks for all devices in the domain */ -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks);if (!IS_ERR (domain->regulator)) { ret =regulator_disable (domain->regulator);
@@ -479,13 +529,17 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd) } } + domain->enabled--; +pm_runtime_put_sync_suspend (domain->dev);return 0 ; out_clk_disable:if (!domain->keep_clocks) -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); + + domain->enabled--;return ret; }
@@ -1514,8 +1568,12 @@ static int imx_gpcv2_probe(struct platform_device *pdev) domain->regmap = regmap; domain->regs = domain_data->pgc_regs; - domain->genpd.power_on = imx_pgc_power_up; - domain->genpd.power_off = imx_pgc_power_down; + domain->genpd.power_pre_on = imx_pgc_power_pre_up; + domain->genpd.power_on = imx_pgc_power_up; + domain->genpd.power_post_on = imx_pgc_power_post_up; + domain->genpd.power_off_pre = imx_pgc_power_down_pre; + domain->genpd.power_off = imx_pgc_power_down; + domain->genpd.power_off_post = imx_pgc_power_down_post; pd_pdev->dev.parent = dev; pd_pdev->dev.of_node = np;
Signature · 1 line
2.35.1
Re: [PATCH 2/3] [RFC] soc: imx: gpcv2: Split clock prepare from clock enable in the domain
Hi Marek,
Quoted 14 lines · Marek Vasut
It is possible for
clk_disable_unused()to trigger lockdep warning regarding lock ordering in this driver. This happens in case of the following conditions:A) clock core
clk_disable_unused()triggers the following sequence in a driver which also uses GPCv2 domain:
-clk_prepare_lock()-> obtains clock coreprepare_lock
- pm_runtime_get*() -> obtains&blk_ctrl_genpd_lock_classB) driver powers up a power domain and triggers the following sequence in GPCv2:
-pm_runtime_get_sync()-> obtains&blk_ctrl_genpd_lock_class
-clk_bulk_prepare_enable()-> obtains clock coreprepare_lock
Quoted 7 lines
This can lead to a deadlock in case A and B runs on separate CPUs.
To avoid the deadlock, split clk_*prepare() from clk_*enable() and call the former in
power_pre_on()callback, beforepm_runtime_get_sync(). The reverse is implemented in thepower_off_post()callback in the same way. This way, the GPCv2 driver always claims theprepare_lockbeforeblk_ctrl_genpd_lock_classand the deadlock is avoided.
How about using notifier such as, GENPD_NOTIFY_PRE_ON?
Regards,
Quoted 177 lines
Signed-off-by Marek Vasut <[email protected]>---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]--- drivers/soc/imx/gpcv2.c | 74 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 8 deletions(-)drivers/soc/imx/gpcv2.c +66
-8@@ -298,6 +298,8 @@ struct imx_pgc_domain {unsigned int pgc_sw_pup_reg;unsigned int pgc_sw_pdn_reg; + +int enabled; };struct imx_pgc_domain_data {@@ -313,6 +315,52 @@ to_imx_pgc_domain(struct generic_pm_domain *genpd) return container_of(genpd, struct imx_pgc_domain, genpd); } +static int imx_pgc_power_pre_up (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); +int ret; + + ret =clk_bulk_prepare (domain->num_clks, domain->clks); +if (ret) +dev_err (domain->dev,"failed to prepare reset clocks\n" ); + +return ret; +} + +static int imx_pgc_power_post_up (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); + +if (!domain->keep_clocks && domain->enabled) +clk_bulk_unprepare (domain->num_clks, domain->clks); + +return 0 ; +} + +static int imx_pgc_power_down_pre (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); +int ret; + +if (!domain->keep_clocks || !domain->enabled) { + ret =clk_bulk_prepare (domain->num_clks, domain->clks); +if (ret) +dev_err (domain->dev,"failed to prepare reset clocks\n" ); + } + +return ret; +} + +static int imx_pgc_power_down_post (struct generic_pm_domain *genpd) +{ +struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd); + +if (!domain->keep_clocks || !domain->enabled) +clk_bulk_unprepare (domain->num_clks, domain->clks); + +return 0 ; +} +static int imx_pgc_power_up (struct generic_pm_domain *genpd) {struct imx_pgc_domain *domain =to_imx_pgc_domain (genpd);@@ -338,7 +386,7 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)reset_control_assert (domain->reset);/* Enable reset clocks for all devices in the domain */ - ret =clk_bulk_prepare_enable (domain->num_clks, domain->clks); + ret =clk_bulk_enable (domain->num_clks, domain->clks);if (ret) {dev_err (domain->dev,"failed to enable reset clocks\n" ); goto out_regulator_disable;@@ -397,12 +445,14 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)/* Disable reset clocks for all devices in the domain */ if (!domain->keep_clocks) -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); + + domain->enabled++;return 0 ; out_clk_disable: -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); out_regulator_disable:if (!IS_ERR (domain->regulator))regulator_disable (domain->regulator);@@ -420,7 +470,7 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd)/* Enable reset clocks for all devices in the domain */ if (!domain->keep_clocks) { - ret =clk_bulk_prepare_enable (domain->num_clks, domain->clks); + ret =clk_bulk_enable (domain->num_clks, domain->clks);if (ret) {dev_err (domain->dev,"failed to enable reset clocks\n" );return ret;@@ -467,7 +517,7 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd) }/* Disable reset clocks for all devices in the domain */ -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks);if (!IS_ERR (domain->regulator)) { ret =regulator_disable (domain->regulator);@@ -479,13 +529,17 @@ static int imx_pgc_power_down(struct generic_pm_domain *genpd) } } + domain->enabled--; +pm_runtime_put_sync_suspend (domain->dev);return 0 ; out_clk_disable:if (!domain->keep_clocks) -clk_bulk_disable_unprepare (domain->num_clks, domain->clks); +clk_bulk_disable (domain->num_clks, domain->clks); + + domain->enabled--;return ret; }@@ -1514,8 +1568,12 @@ static int imx_gpcv2_probe(struct platform_device *pdev) domain->regmap = regmap; domain->regs = domain_data->pgc_regs; - domain->genpd.power_on = imx_pgc_power_up; - domain->genpd.power_off = imx_pgc_power_down; + domain->genpd.power_pre_on = imx_pgc_power_pre_up; + domain->genpd.power_on = imx_pgc_power_up; + domain->genpd.power_post_on = imx_pgc_power_post_up; + domain->genpd.power_off_pre = imx_pgc_power_down_pre; + domain->genpd.power_off = imx_pgc_power_down; + domain->genpd.power_off_post = imx_pgc_power_down_post; pd_pdev->dev.parent = dev; pd_pdev->dev.of_node = np;
RFC PATCH 3/3 soc: imx: imx8m-blk-ctrl: Split clock prepare from clock enable in the domain
In reply to Marek VasutIt is possible for clk_disable_unused() to trigger lockdep warning regarding lock ordering in this driver. This happens in case of the following conditions:
A) clock core clk_disable_unused() triggers the following sequence in a driver which also uses blkctrl domain:
- clk_prepare_lock() -> obtains clock core prepare_lock
- pm_runtime_get*() -> obtains &blk_ctrl_genpd_lock_class
B) driver powers up a power domain and triggers the following sequence in blkctrl:
- pm_runtime_get_sync() -> obtains &blk_ctrl_genpd_lock_class
- clk_bulk_prepare_enable() -> obtains clock core prepare_lock
This can lead to a deadlock in case A and B runs on separate CPUs.
To avoid the deadlock, split clk_*prepare() from clk_*enable() and call the former in power_pre_on() callback, before pm_runtime_get_sync(). The reverse is implemented in the power_off_post() callback in the same way. This way, the blkctrl driver always claims the prepare_lock before blk_ctrl_genpd_lock_class and the deadlock is avoided.
---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]
--- drivers/soc/imx/imx8mp-blk-ctrl.c | 38 +++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-)
drivers/soc/imx/imx8mp-blk-ctrl.c +33 -5
@@ -408,6 +408,30 @@ static const struct imx8mp_blk_ctrl_data imx8mp_hdmi_blk_ctl_dev_data = { .num_domains = ARRAY_SIZE(imx8mp_hdmi_domain_data), }; +static int imx8mp_blk_ctrl_power_pre_on (struct generic_pm_domain *genpd) +{ +struct imx8mp_blk_ctrl_domain *domain =to_imx8mp_blk_ctrl_domain (genpd); +const struct imx8mp_blk_ctrl_domain_data *data = domain->data; +struct imx8mp_blk_ctrl *bc = domain->bc; +int ret; + + ret =clk_bulk_prepare (data->num_clks, domain->clks); +if (ret) +dev_err (bc->dev,"failed to enable clocks\n" ); + +return ret; +} + +static int imx8mp_blk_ctrl_power_off_post (struct generic_pm_domain *genpd) +{ +struct imx8mp_blk_ctrl_domain *domain =to_imx8mp_blk_ctrl_domain (genpd); +const struct imx8mp_blk_ctrl_domain_data *data = domain->data; + +clk_bulk_unprepare (data->num_clks, domain->clks); + +return 0 ; +} +static int imx8mp_blk_ctrl_power_on (struct generic_pm_domain *genpd) {struct imx8mp_blk_ctrl_domain *domain =to_imx8mp_blk_ctrl_domain (genpd);
@@ -423,7 +447,7 @@ static int imx8mp_blk_ctrl_power_on(struct generic_pm_domain *genpd) }/* enable upstream clocks */ - ret =clk_bulk_prepare_enable (data->num_clks, domain->clks); + ret =clk_bulk_enable (data->num_clks, domain->clks);if (ret) {dev_err (bc->dev,"failed to enable clocks\n" ); goto bus_put;
@@ -443,12 +467,12 @@ static int imx8mp_blk_ctrl_power_on(struct generic_pm_domain *genpd)if (ret)dev_err (bc->dev,"failed to set icc bw\n" ); -clk_bulk_disable_unprepare (data->num_clks, domain->clks); +clk_bulk_disable (data->num_clks, domain->clks);return 0 ; clk_disable: -clk_bulk_disable_unprepare (data->num_clks, domain->clks); +clk_bulk_disable (data->num_clks, domain->clks); bus_put:pm_runtime_put (bc->bus_power_dev);
@@ -462,7 +486,7 @@ static int imx8mp_blk_ctrl_power_off(struct generic_pm_domain *genpd)struct imx8mp_blk_ctrl *bc = domain->bc;int ret; - ret =clk_bulk_prepare_enable (data->num_clks, domain->clks); + ret =clk_bulk_enable (data->num_clks, domain->clks);if (ret) {dev_err (bc->dev,"failed to enable clocks\n" );return ret;
@@ -471,7 +495,7 @@ static int imx8mp_blk_ctrl_power_off(struct generic_pm_domain *genpd)/* domain specific blk-ctrl manipulation */ bc->power_off (bc, domain); -clk_bulk_disable_unprepare (data->num_clks, domain->clks); +clk_bulk_disable (data->num_clks, domain->clks);/* power down upstream GPC domain */ pm_runtime_put (domain->power_dev);
@@ -585,8 +609,12 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev)dev_set_name (domain->power_dev,"%s" , data->name); domain->genpd.name = data->name; + domain->genpd.power_pre_on = imx8mp_blk_ctrl_power_pre_on; domain->genpd.power_on = imx8mp_blk_ctrl_power_on; + domain->genpd.power_post_on = imx8mp_blk_ctrl_power_off_post; + domain->genpd.power_off_pre = imx8mp_blk_ctrl_power_pre_on; domain->genpd.power_off = imx8mp_blk_ctrl_power_off; + domain->genpd.power_off_post = imx8mp_blk_ctrl_power_off_post; domain->bc = bc; domain->id = i;
Signature · 1 line
2.35.1
Hi Marek,
Thank you for the patch.
Quoted 225 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.Signed-off-by Marek Vasut <[email protected]>---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]--- drivers/base/power/domain.c | 103 ++++++++++++++++++++++++++++++++---- include/linux/pm_domain.h | 4 ++ 2 files changed, 97 insertions(+), 10 deletions(-)drivers/base/power/domain.c +74
-8@@ -494,6 +494,22 @@ void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) }EXPORT_SYMBOL_GPL (dev_pm_genpd_set_next_wakeup); +static int genpd_power_pre_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_pre_on) +return 0 ; + +return genpd->power_pre_on (genpd); +} + +static int genpd_power_post_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_post_on) +return 0 ; + +return genpd->power_post_on (genpd); +} +static int _genpd_power_on (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;@@ -544,6 +560,22 @@ static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)return ret; } +static int genpd_power_off_pre (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_pre) +return 0 ; + +return genpd->power_off_pre (genpd); +} + +static int genpd_power_off_post (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_post) +return 0 ; + +return genpd->power_off_post (genpd); +} +static int _genpd_power_off (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;@@ -816,12 +848,18 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,static void genpd_power_off_work_fn (struct work_struct *work) {struct generic_pm_domain *genpd; +int ret; genpd = container_of(work, struct generic_pm_domain, power_off_work); + ret =genpd_power_off_pre (genpd); +if (ret) +return ;genpd_lock (genpd);genpd_power_off (genpd, false,0 );genpd_unlock (genpd); + ret =genpd_power_off_post (genpd); +WARN_ON_ONCE (ret); }/** @@ -938,12 +976,14 @@ static int genpd_runtime_suspend(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd))return 0 ; + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); gpd_data->rpm_pstate =genpd_drop_performance_state (dev);genpd_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/** @@ -977,12 +1017,21 @@ static int genpd_runtime_resume(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd)) goto out; + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd); ret =genpd_power_on (genpd,0 );if (!ret)genpd_restore_performance_state (dev, gpd_data->rpm_pstate);genpd_unlock (genpd); +if (ret) { +genpd_power_post_on (genpd); +return ret; + } + + ret =genpd_power_post_on (genpd);if (ret)return ret;@@ -1017,10 +1066,13 @@ static int genpd_runtime_resume(struct device *dev)genpd_stop_dev (genpd, dev); err_poweroff:if (!pm_runtime_is_irq_safe (dev) ||genpd_is_irq_safe (genpd)) { -genpd_lock (genpd); - gpd_data->rpm_pstate =genpd_drop_performance_state (dev); -genpd_power_off (genpd, true,0 ); -genpd_unlock (genpd); +if (!genpd_power_off_pre (genpd)) { +genpd_lock (genpd); + gpd_data->rpm_pstate =genpd_drop_performance_state (dev); +genpd_power_off (genpd, true,0 ); +genpd_unlock (genpd); +genpd_power_off_post (genpd); + } }return ret;@@ -1225,12 +1277,14 @@ static int genpd_finish_suspend(struct device *dev, bool poweroff) } } + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); genpd->suspended_count++;genpd_sync_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/** @@ -1267,10 +1321,16 @@ static int genpd_resume_noirq(struct device *dev)if (device_wakeup_path (dev) &&genpd_is_active_wakeup (genpd))return pm_generic_resume_noirq (dev); + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);genpd_sync_power_on (genpd, true,0 ); genpd->suspended_count--;genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {@@ -1378,6 +1438,9 @@ static int genpd_restore_noirq(struct device *dev)* At this point suspended_count == 0 means we are being run for the * first time for the given domain in the present cycle. */ + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);if (genpd->suspended_count++ ==0 ) {/* @@ -1390,6 +1453,9 @@ static int genpd_restore_noirq(struct device *dev)genpd_sync_power_on (genpd, true,0 );genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {
drivers/base/power/domain.c static int genpd_restore_noirq(struct device *dev) · line 1482
Quoted 4 lines
drivers/base/power/domain.c +1
-0@@ -1413,6 +1479,7 @@ static int genpd_restore_noirq(struct device *dev)static void genpd_complete (struct device *dev) {struct generic_pm_domain *genpd; +int ret;
This variable is unused, causing a compilation error.
Quoted 73 lines
drivers/base/power/domain.c +18
-2dev_dbg (dev,"%s()\n" , __func__);@@ -1435,6 +1502,7 @@ static void genpd_switch_state(struct device *dev, bool suspend) {struct generic_pm_domain *genpd;bool use_lock; +int ret; genpd =dev_to_genpd_safe (dev);if (!genpd)@@ -1442,8 +1510,13 @@ static void genpd_switch_state(struct device *dev, bool suspend) use_lock =genpd_is_irq_safe (genpd); -if (use_lock) +if (use_lock) { + ret = suspend ?genpd_power_off_pre (genpd) : +genpd_power_pre_on (genpd); +if (ret) +return ;genpd_lock (genpd); + }if (suspend) { genpd->suspended_count++;@@ -1453,8 +1526,12 @@ static void genpd_switch_state(struct device *dev, bool suspend) genpd->suspended_count--; } -if (use_lock) +if (use_lock) {genpd_unlock (genpd); + ret = suspend ?genpd_power_off_post (genpd) : +genpd_power_post_on (genpd); +WARN_ON_ONCE (ret); + } }/** @@ -2750,9 +2827,15 @@ static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev, dev->pm_domain->sync = genpd_dev_pm_sync;if (power_on) { + ret =genpd_power_pre_on (pd); +if (ret) +return ret;genpd_lock (pd); ret =genpd_power_on (pd,0 );genpd_unlock (pd); + ret =genpd_power_post_on (pd); +if (ret) +return ret; }if (ret) {include/linux/pm_domain.h +4
-0@@ -134,8 +134,12 @@ struct generic_pm_domain {unsigned int prepared_count;/* Suspend counter of prepared devices */ unsigned int performance_state;/* Aggregated max performance state */ cpumask_var_t cpus;/* A cpumask of the attached CPUs */ +int (*power_off_pre)(struct generic_pm_domain *domain);int (*power_off)(struct generic_pm_domain *domain); +int (*power_off_post)(struct generic_pm_domain *domain); +int (*power_pre_on)(struct generic_pm_domain *domain);int (*power_on)(struct generic_pm_domain *domain); +int (*power_post_on)(struct generic_pm_domain *domain);struct raw_notifier_head power_notifiers;/* Power on/off notifiers */ struct opp_table *opp_table;/* OPP table of the genpd */ unsigned int (*opp_to_performance_state)(struct generic_pm_domain *genpd,
Signature · 3 lines
Regards, Laurent Pinchart
On 11/9/22 14:19, Laurent Pinchart wrote:
Hi,
[...]
Quoted 7 lines
drivers/base/power/domain.c static int genpd_restore_noirq(struct device *dev) · line 1482
Quoted 5 lines
drivers/base/power/domain.c +1
-0@@ -1413,6 +1479,7 @@ static int genpd_restore_noirq(struct device *dev)static void genpd_complete (struct device *dev) {struct generic_pm_domain *genpd; +int ret;This variable is unused, causing a compilation error.
I suspect only with -Werror, but anyway, already fixed locally, it's a rebase on latest next artifact.
[...]
Quoted 9 lines
Quoted 9 lines
include/linux/pm_domain.h +1
-0@@ -134,8 +134,12 @@ struct generic_pm_domain {unsigned int prepared_count;/* Suspend counter of prepared devices */ unsigned int performance_state;/* Aggregated max performance state */ cpumask_var_t cpus;/* A cpumask of the attached CPUs */ +int (*power_off_pre)(struct generic_pm_domain *domain);
Quoted 8 lines
include/linux/pm_domain.h struct generic_pm_domain { · line 145
Quoted 8 lines
include/linux/pm_domain.h +3
-0int (*power_off)(struct generic_pm_domain *domain); +int (*power_off_post)(struct generic_pm_domain *domain); +int (*power_pre_on)(struct generic_pm_domain *domain);int (*power_on)(struct generic_pm_domain *domain); +int (*power_post_on)(struct generic_pm_domain *domain);struct raw_notifier_head power_notifiers;/* Power on/off notifiers */ struct opp_table *opp_table;/* OPP table of the genpd */ unsigned int (*opp_to_performance_state)(struct generic_pm_domain *genpd,
I am looking more for a feedback on this extension of the callbacks, and on the overall approach. Is this something which looks OK, or would there be better way to handle this ?
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.
I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
Quoted 7 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.
To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in imx8mp_blk_ctrl_probe() that seems to be pointing in this direction too.
"* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Kind regards
Uffe
Quoted 295 lines
Signed-off-by Marek Vasut <[email protected]>---
Cc: Adam Ford <[email protected]>
Cc: Fabio Estevam <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Jacky Bai <[email protected]>
Cc: Kevin Hilman <[email protected]>
Cc: Laurent Pinchart <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Liam Girdwood <[email protected]>
Cc: Lucas Stach <[email protected]>
Cc: Marek Vasut <[email protected]>
Cc: Mark Brown <[email protected]>
Cc: Martin Kepplinger <[email protected]>
Cc: Pavel Machek <[email protected]>
Cc: Peng Fan <[email protected]>
Cc: Pengutronix Kernel Team <[email protected]>
Cc: Philipp Zabel <[email protected]>
Cc: Rafael J. Wysocki <[email protected]>
Cc: Sascha Hauer <[email protected]>
Cc: Shawn Guo <[email protected]>
Cc: Shengjiu Wang <[email protected]>
Cc: Stephen Boyd <[email protected]>
Cc: Ulf Hansson <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
To: [email protected]--- drivers/base/power/domain.c | 103 ++++++++++++++++++++++++++++++++---- include/linux/pm_domain.h | 4 ++ 2 files changed, 97 insertions(+), 10 deletions(-)drivers/base/power/domain.c +93
-10@@ -494,6 +494,22 @@ void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) }EXPORT_SYMBOL_GPL (dev_pm_genpd_set_next_wakeup); +static int genpd_power_pre_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_pre_on) +return 0 ; + +return genpd->power_pre_on (genpd); +} + +static int genpd_power_post_on (struct generic_pm_domain *genpd) +{ +if (!genpd->power_post_on) +return 0 ; + +return genpd->power_post_on (genpd); +} +static int _genpd_power_on (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;@@ -544,6 +560,22 @@ static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)return ret; } +static int genpd_power_off_pre (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_pre) +return 0 ; + +return genpd->power_off_pre (genpd); +} + +static int genpd_power_off_post (struct generic_pm_domain *genpd) +{ +if (!genpd->power_off_post) +return 0 ; + +return genpd->power_off_post (genpd); +} +static int _genpd_power_off (struct generic_pm_domain *genpd,bool timed) {unsigned int state_idx = genpd->state_idx;@@ -816,12 +848,18 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,static void genpd_power_off_work_fn (struct work_struct *work) {struct generic_pm_domain *genpd; +int ret; genpd = container_of(work, struct generic_pm_domain, power_off_work); + ret =genpd_power_off_pre (genpd); +if (ret) +return ;genpd_lock (genpd);genpd_power_off (genpd, false,0 );genpd_unlock (genpd); + ret =genpd_power_off_post (genpd); +WARN_ON_ONCE (ret); }/** @@ -938,12 +976,14 @@ static int genpd_runtime_suspend(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd))return 0 ; + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); gpd_data->rpm_pstate =genpd_drop_performance_state (dev);genpd_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/** @@ -977,12 +1017,21 @@ static int genpd_runtime_resume(struct device *dev)if (irq_safe_dev_in_sleep_domain (dev, genpd)) goto out; + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd); ret =genpd_power_on (genpd,0 );if (!ret)genpd_restore_performance_state (dev, gpd_data->rpm_pstate);genpd_unlock (genpd); +if (ret) { +genpd_power_post_on (genpd); +return ret; + } + + ret =genpd_power_post_on (genpd);if (ret)return ret;@@ -1017,10 +1066,13 @@ static int genpd_runtime_resume(struct device *dev)genpd_stop_dev (genpd, dev); err_poweroff:if (!pm_runtime_is_irq_safe (dev) ||genpd_is_irq_safe (genpd)) { -genpd_lock (genpd); - gpd_data->rpm_pstate =genpd_drop_performance_state (dev); -genpd_power_off (genpd, true,0 ); -genpd_unlock (genpd); +if (!genpd_power_off_pre (genpd)) { +genpd_lock (genpd); + gpd_data->rpm_pstate =genpd_drop_performance_state (dev); +genpd_power_off (genpd, true,0 ); +genpd_unlock (genpd); +genpd_power_off_post (genpd); + } }return ret;@@ -1225,12 +1277,14 @@ static int genpd_finish_suspend(struct device *dev, bool poweroff) } } + ret =genpd_power_off_pre (genpd); +if (ret) +return ret;genpd_lock (genpd); genpd->suspended_count++;genpd_sync_power_off (genpd, true,0 );genpd_unlock (genpd); - -return 0 ; +return genpd_power_off_post (genpd); }/** @@ -1267,10 +1321,16 @@ static int genpd_resume_noirq(struct device *dev)if (device_wakeup_path (dev) &&genpd_is_active_wakeup (genpd))return pm_generic_resume_noirq (dev); + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);genpd_sync_power_on (genpd, true,0 ); genpd->suspended_count--;genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {@@ -1378,6 +1438,9 @@ static int genpd_restore_noirq(struct device *dev)* At this point suspended_count == 0 means we are being run for the * first time for the given domain in the present cycle. */ + ret =genpd_power_pre_on (genpd); +if (ret) +return ret;genpd_lock (genpd);if (genpd->suspended_count++ ==0 ) {/* @@ -1390,6 +1453,9 @@ static int genpd_restore_noirq(struct device *dev)genpd_sync_power_on (genpd, true,0 );genpd_unlock (genpd); + ret =genpd_power_post_on (genpd); +if (ret) +return ret;if (genpd->dev_ops.stop && genpd->dev_ops.start && !pm_runtime_status_suspended (dev)) {@@ -1413,6 +1479,7 @@ static int genpd_restore_noirq(struct device *dev)static void genpd_complete (struct device *dev) {struct generic_pm_domain *genpd; +int ret;dev_dbg (dev,"%s()\n" , __func__);@@ -1435,6 +1502,7 @@ static void genpd_switch_state(struct device *dev, bool suspend) {struct generic_pm_domain *genpd;bool use_lock; +int ret; genpd =dev_to_genpd_safe (dev);if (!genpd)@@ -1442,8 +1510,13 @@ static void genpd_switch_state(struct device *dev, bool suspend) use_lock =genpd_is_irq_safe (genpd); -if (use_lock) +if (use_lock) { + ret = suspend ?genpd_power_off_pre (genpd) : +genpd_power_pre_on (genpd); +if (ret) +return ;genpd_lock (genpd); + }if (suspend) { genpd->suspended_count++;@@ -1453,8 +1526,12 @@ static void genpd_switch_state(struct device *dev, bool suspend) genpd->suspended_count--; } -if (use_lock) +if (use_lock) {genpd_unlock (genpd); + ret = suspend ?genpd_power_off_post (genpd) : +genpd_power_post_on (genpd); +WARN_ON_ONCE (ret); + } }/** @@ -2750,9 +2827,15 @@ static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev, dev->pm_domain->sync = genpd_dev_pm_sync;if (power_on) { + ret =genpd_power_pre_on (pd); +if (ret) +return ret;genpd_lock (pd); ret =genpd_power_on (pd,0 );genpd_unlock (pd); + ret =genpd_power_post_on (pd); +if (ret) +return ret; }if (ret) {include/linux/pm_domain.h +4
-1@@ -134,8 +134,12 @@ struct generic_pm_domain {unsigned int prepared_count;/* Suspend counter of prepared devices */ unsigned int performance_state;/* Aggregated max performance state */ cpumask_var_t cpus;/* A cpumask of the attached CPUs */ +int (*power_off_pre)(struct generic_pm_domain *domain);int (*power_off)(struct generic_pm_domain *domain); +int (*power_off_post)(struct generic_pm_domain *domain); +int (*power_pre_on)(struct generic_pm_domain *domain);int (*power_on)(struct generic_pm_domain *domain); +int (*power_post_on)(struct generic_pm_domain *domain);struct raw_notifier_head power_notifiers;/* Power on/off notifiers */ struct opp_table *opp_table;/* OPP table of the genpd */ unsigned int (*opp_to_performance_state)(struct generic_pm_domain *genpd, --2.35.1
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 13 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too.
Quoted 6 lines
"* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
https://lore.kernel.org/all/[email protected]/
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
[...]
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"
Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems the clk_disable_unused() thingy, would trigger a similar problem for this configuration for the imx8mp platform.
Case #1:
Triggered from the workqueue, the genpd_power_off_work_fn() ends up calling clk_bulk_unprepare(), from a genpd's ->power_off() callback(), which has been assigned to imx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock via clk_bulk_unprepare() while holding the genpd-lock(s).
Case #0:
The "drm driver" calls clk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, the clk_set_rate() path continues to call clk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call to clk_pm_runtime_get() ends up in genpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to call genpd_power_on() to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.
The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.
Kind regards
Uffe
[1]
https://lore.kernel.org/all/[email protected]/
https://lore.kernel.org/all/CAE-0n52xbZeJ66RaKwggeRB57fUAwjvxGxfFMKOKJMKVyFTe+w@mail.gmail.com/
Hi Ulf,
Am Mittwoch, dem 16.11.2022 um 13:41 +0100 schrieb Ulf Hansson:
Quoted 82 lines
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning
clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems theclk_disable_unused()thingy, would trigger a similar problem for this configuration for the imx8mp platform.Case #1:
Triggered from the workqueue, thegenpd_power_off_work_fn()ends up callingclk_bulk_unprepare(), from a genpd's ->power_off()callback(), which has been assigned toimx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock viaclk_bulk_unprepare()while holding the genpd-lock(s).Case #0:
The "drm driver" callsclk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, theclk_set_rate()path continues to callclk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call toclk_pm_runtime_get()ends up ingenpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to callgenpd_power_on()to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
Quoted 6 lines
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.
The dependency chain isn't circular, it just happens to converge in the clock framework and its single big hammer lock. The chain looks some thing like this:
1. DRM driver request pixel clock (clk_prepare_enable -> clk_prepare_mutex)
2. Pixel clock is supplied from the PHY, which is in a power domain, so in order to supply the clock it needs to runtime resume
3. genpd powers up the PHY blk-ctrl domain, which again is inside a GPCv2 power domain
4. genpd powers up GPCv2 domain, which needs a specific clock to be running in order to power up the domain, so it does a clk_prepare_enable, which now happens to hit the clk_prepare_mutex taken in step 1.
As the runtime resume/suspend for the PHY may go through a workqueue we have two different contexts trying to take the clk_prepare_mutex, which is what lockdep complains about.
Quoted 9 lines
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.
I don't see any straight forward way to avoid the clock framework calls in the chain laid out above. I would be happy if anyone has some good suggestions.
Regards,
Lucas
Quoted 108 lines · Lucas Stach
Hi Ulf,
Am Mittwoch, dem 16.11.2022 um 13:41 +0100 schrieb Ulf Hansson:
Quoted 82 lines
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning
clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems theclk_disable_unused()thingy, would trigger a similar problem for this configuration for the imx8mp platform.Case #1:
Triggered from the workqueue, thegenpd_power_off_work_fn()ends up callingclk_bulk_unprepare(), from a genpd's ->power_off()callback(), which has been assigned toimx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock viaclk_bulk_unprepare()while holding the genpd-lock(s).Case #0:
The "drm driver" callsclk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, theclk_set_rate()path continues to callclk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call toclk_pm_runtime_get()ends up ingenpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to callgenpd_power_on()to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
Quoted 6 lines
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.The dependency chain isn't circular, it just happens to converge in the clock framework and its single big hammer lock. The chain looks some thing like this:
1. DRM driver request pixel clock (
clk_prepare_enable->clk_prepare_mutex)
2. Pixel clock is supplied from the PHY, which is in a power domain, so in order to supply the clock it needs to runtime resume
3. genpd powers up the PHY blk-ctrl domain, which again is inside a GPCv2 power domain
4. genpd powers up GPCv2 domain, which needs a specific clock to be running in order to power up the domain, so it does aclk_prepare_enable, which now happens to hit theclk_prepare_mutextaken in step 1.
Quoted 3 lines
As the runtime resume/suspend for the PHY may go through a workqueue we have two different contexts trying to take the
clk_prepare_mutex, which is what lockdep complains about.
I see. Thanks for bringing some more clarity in this.
So the above is described in some of the in-kernel DTS(i) files too?
Quoted 11 lines
Quoted 10 lines
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.
Quoted 3 lines
I don't see any straight forward way to avoid the clock framework calls in the chain laid out above. I would be happy if anyone has some good suggestions.
I think you misunderstood what I proposed above. What I was suggesting is that we could re-work the runtime PM deployment in the clock framework.
In this way, we would not be holding the global clk-prepare lock while trying to power-on the phy and its PM domain. Wouldn't that work?
Kind regards
Uffe
Hi Ulf,
Quoted 132 lines · Ulf Hansson
Quoted 108 lines · Lucas Stach
Hi Ulf,
Am Mittwoch, dem 16.11.2022 um 13:41 +0100 schrieb Ulf Hansson:
Quoted 82 lines
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning
clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems theclk_disable_unused()thingy, would trigger a similar problem for this configuration for the imx8mp platform.Case #1:
Triggered from the workqueue, thegenpd_power_off_work_fn()ends up callingclk_bulk_unprepare(), from a genpd's ->power_off()callback(), which has been assigned toimx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock viaclk_bulk_unprepare()while holding the genpd-lock(s).Case #0:
The "drm driver" callsclk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, theclk_set_rate()path continues to callclk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call toclk_pm_runtime_get()ends up ingenpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to callgenpd_power_on()to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
Quoted 6 lines
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.The dependency chain isn't circular, it just happens to converge in the clock framework and its single big hammer lock. The chain looks some thing like this:
1. DRM driver request pixel clock (
clk_prepare_enable->clk_prepare_mutex)
2. Pixel clock is supplied from the PHY, which is in a power domain, so in order to supply the clock it needs to runtime resume
3. genpd powers up the PHY blk-ctrl domain, which again is inside a GPCv2 power domain
4. genpd powers up GPCv2 domain, which needs a specific clock to be running in order to power up the domain, so it does aclk_prepare_enable, which now happens to hit theclk_prepare_mutextaken in step 1.Quoted 3 lines
As the runtime resume/suspend for the PHY may go through a workqueue we have two different contexts trying to take the
clk_prepare_mutex, which is what lockdep complains about.I see. Thanks for bringing some more clarity in this.
So the above is described in some of the in-kernel DTS(i) files too?
Quoted 11 lines
Quoted 10 lines
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.Quoted 3 lines
I don't see any straight forward way to avoid the clock framework calls in the chain laid out above. I would be happy if anyone has some good suggestions.
Quoted 6 lines
I think you misunderstood what I proposed above. What I was suggesting is that we could re-work the runtime PM deployment in the clock framework.
In this way, we would not be holding the global clk-prepare lock while trying to power-on the phy and its PM domain. Wouldn't that work?
Do you have time to give a look at NXP downstream patch to address the lock issue in genpd side [1]?
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
[1]https://github.com/nxp-imx/linux-imx/commit/c85126180a10439a8c7db1800529b4857d91c609
Thanks,
Peng.
Quoted 3 lines
Kind regards
Uffe
Quoted 140 lines · Peng Fan
Hi Ulf,
Quoted 136 lines · Ulf Hansson
Quoted 108 lines · Lucas Stach
Hi Ulf,
Am Mittwoch, dem 16.11.2022 um 13:41 +0100 schrieb Ulf Hansson:
Quoted 82 lines
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning
clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems theclk_disable_unused()thingy, would trigger a similar problem for this configuration for the imx8mp platform.Case #1:
Triggered from the workqueue, thegenpd_power_off_work_fn()ends up callingclk_bulk_unprepare(), from a genpd's ->power_off()callback(), which has been assigned toimx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock viaclk_bulk_unprepare()while holding the genpd-lock(s).Case #0:
The "drm driver" callsclk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, theclk_set_rate()path continues to callclk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call toclk_pm_runtime_get()ends up ingenpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to callgenpd_power_on()to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
Quoted 6 lines
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.The dependency chain isn't circular, it just happens to converge in the clock framework and its single big hammer lock. The chain looks some thing like this:
1. DRM driver request pixel clock (
clk_prepare_enable->clk_prepare_mutex)
2. Pixel clock is supplied from the PHY, which is in a power domain, so in order to supply the clock it needs to runtime resume
3. genpd powers up the PHY blk-ctrl domain, which again is inside a GPCv2 power domain
4. genpd powers up GPCv2 domain, which needs a specific clock to be running in order to power up the domain, so it does aclk_prepare_enable, which now happens to hit theclk_prepare_mutextaken in step 1.Quoted 3 lines
As the runtime resume/suspend for the PHY may go through a workqueue we have two different contexts trying to take the
clk_prepare_mutex, which is what lockdep complains about.I see. Thanks for bringing some more clarity in this.
So the above is described in some of the in-kernel DTS(i) files too?
Quoted 11 lines
Quoted 10 lines
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.Quoted 3 lines
I don't see any straight forward way to avoid the clock framework calls in the chain laid out above. I would be happy if anyone has some good suggestions.
I think you misunderstood what I proposed above. What I was suggesting is that we could re-work the runtime PM deployment in the clock framework.
Quoted 6 lines
Quoted 2 lines
In this way, we would not be holding the global clk-prepare lock while trying to power-on the phy and its PM domain. Wouldn't that work?
Do you have time to give a look at NXP downstream patch to address the lock issue in genpd side [1]?
Thanks for sharing this! I had a look and the approach seems like it could work for you. However, I am still not sure it's the correct thing to do.
Quoted 3 lines
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
I understand. Let me try to prepare a hackish patch that we can use to run some tests.
Unfortunately, I can't make any promises for when, but I will do my best.
Quoted 8 lines
[1]https://github.com/nxp-imx/linux-imx/commit/c85126180a10439a8c7db1800529b4857d91c609
Thanks,
Peng.Quoted 3 lines
Kind regards
Uffe
Kind regards
Uffe
Quoted 153 lines · Ulf Hansson
Quoted 140 lines · Peng Fan
Hi Ulf,
Quoted 136 lines · Ulf Hansson
Quoted 108 lines · Lucas Stach
Hi Ulf,
Am Mittwoch, dem 16.11.2022 um 13:41 +0100 schrieb Ulf Hansson:
Quoted 82 lines
+ Stephen Boyd
Quoted 30 lines · Marek Vasut
Quoted 8 lines · Ulf Hansson
Quoted 4 lines · Marek Vasut
Currently it is possible that a power domain power on or off would claim the genpd lock first and clock core
prepare_locksecond, while another thread could do the reverse, and this would trigger lockdep warning.I am not quite sure I fully understand. In this case is the lockdep warning relevant or just something that we want to silence?
This is a valid problem, see patches 2/3 and 3/3 for details too.
Quoted 17 lines
Quoted 6 lines
Introduce new callbacks, .power_pre/post_on() and .power_off_pre/post(), which are triggered before the
genpd_lock()and aftergenpd_unlock()respectively in case the domain is powered on and off. Those are meant to let drivers claim clock coreprepare_lockvia clk_*prepare() call and release the lock via clk_*unprepare() call to always assure that the clock and genpd lock ordering is correct.To me, this sounds like a problem that may be better fixed by trying to model the parent/child-domains in a more strict way, through genpd.
There is a comment in the code in
imx8mp_blk_ctrl_probe()that seems to be pointing in this direction too."* We use runtime PM to trigger power on/off of the upstream GPC
* domain, as a strict hierarchical parent/child power domain
* setup doesn't allow us to meet the sequencing requirements......"Quoted 6 lines
Quoted 2 lines
I am wondering about what those "sequencing requirements" are - and whether it could make better sense to fix these issues instead?
Here is the lockdep splat:
Yes, that certainly helped!
Quoted 5 lines
It really is a problem between the clock and genpd subsystem locks, they can be claimed in arbitrary order, see patch 2/3 and 3/3.
I think that might clarify what I am attempting to solve here.
Let me try to put some more words behind this, to make sure I have understood correctly, but also to easier allow more people to chim in.
Note that, in your commit messages in patch2 and patch3, you are mentioning
clk_disable_unused(), but that's not what the lockdep splat above is pointing at. Although, it seems theclk_disable_unused()thingy, would trigger a similar problem for this configuration for the imx8mp platform.Case #1:
Triggered from the workqueue, thegenpd_power_off_work_fn()ends up callingclk_bulk_unprepare(), from a genpd's ->power_off()callback(), which has been assigned toimx8mp_blk_ctrl_power_off(). Before genpd's ->power_off() is called, the genpd-lock(s) have been acquired, thus we are trying to acquire the global clk-prepare lock viaclk_bulk_unprepare()while holding the genpd-lock(s).Case #0:
The "drm driver" callsclk_set_rate(), thus we start by acquiring the global clk-prepare lock. Internally in the clock frameworks, theclk_set_rate()path continues to callclk_pm_runtime_get(). In this case, the corresponding clock provider's struct *device, seems to be attached to a genpd too. This means the call toclk_pm_runtime_get()ends up ingenpd_runtime_resume(), which needs to acquire the genpd-lock(s) before it continues to callgenpd_power_on()to power-on the PM domain. In other words, we try to acquire genpd-lock(s) while holding the global clk-prepare lock.The solution to fix these problems that you suggest in the $subject patch, isn't the direction I want this to take. The new callbacks are prone to be abused and it would also require genpd provider specific code to fix the problems. Of course, we need things to work, but let's look at a couple of options first. See below.
Quoted 6 lines
1)
In a way, it looks like we have a circular description in DT of the hierarchy of the clock- and genpd providers, which is a bit awkward in my opinion. I was browsing the imx8mp DTS files to find this, but I couldn't. Can you perhaps point me to the DTS file(s) you are using? I can try to have a look so see if this could be arranged differently.The dependency chain isn't circular, it just happens to converge in the clock framework and its single big hammer lock. The chain looks some thing like this:
1. DRM driver request pixel clock (
clk_prepare_enable->clk_prepare_mutex)
2. Pixel clock is supplied from the PHY, which is in a power domain, so in order to supply the clock it needs to runtime resume
3. genpd powers up the PHY blk-ctrl domain, which again is inside a GPCv2 power domain
4. genpd powers up GPCv2 domain, which needs a specific clock to be running in order to power up the domain, so it does aclk_prepare_enable, which now happens to hit theclk_prepare_mutextaken in step 1.Quoted 3 lines
As the runtime resume/suspend for the PHY may go through a workqueue we have two different contexts trying to take the
clk_prepare_mutex, which is what lockdep complains about.I see. Thanks for bringing some more clarity in this.
So the above is described in some of the in-kernel DTS(i) files too?
Quoted 11 lines
Quoted 10 lines
2)
What we have seen from other use cases [1], is that calling pm_runtime_get|put*(), while holding subsystem specific locks (like the genpd-lock(s) and clk-prepare lock), isn't working very well. So, I am thinking that we could have a look at the runtime PM deployment in the clock framework, to see if we can avoid holding the global clk-prepare lock, while calling into runtime PM. I believe that should fix these problems too.Quoted 3 lines
I don't see any straight forward way to avoid the clock framework calls in the chain laid out above. I would be happy if anyone has some good suggestions.
I think you misunderstood what I proposed above. What I was suggesting is that we could re-work the runtime PM deployment in the clock framework.
Quoted 6 lines
Quoted 2 lines
In this way, we would not be holding the global clk-prepare lock while trying to power-on the phy and its PM domain. Wouldn't that work?
Do you have time to give a look at NXP downstream patch to address the lock issue in genpd side [1]?
Thanks for sharing this! I had a look and the approach seems like it could work for you. However, I am still not sure it's the correct thing to do.
Quoted 1 line
Quoted 7 lines
Quoted 2 lines
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
I understand. Let me try to prepare a hackish patch that we can use to run some tests.
Unfortunately, I can't make any promises for when, but I will do my best.
Thanks !
Hi Ulf,
On 1/18/2023 8:55 PM, Ulf Hansson wrote:
...
Quoted 9 lines
Quoted 3 lines
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
I understand. Let me try to prepare a hackish patch that we can use to run some tests.
Unfortunately, I can't make any promises for when, but I will do my best.
Have you got time to give a look on this part? Or do you have any suggestions that I could help with?
Thanks,
Peng.
Quoted 11 lines
Quoted 8 lines
[1]https://github.com/nxp-imx/linux-imx/commit/c85126180a10439a8c7db1800529b4857d91c609
Thanks,
Peng.Quoted 3 lines
Kind regards
UffeKind regards
Uffe
Quoted 9 lines · Peng Fan
Hi Ulf,
On 1/18/2023 8:55 PM, Ulf Hansson wrote:
...Quoted 4 lines
Quoted 3 lines
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
Quoted 8 lines
Quoted 5 lines
I understand. Let me try to prepare a hackish patch that we can use to run some tests.
Unfortunately, I can't make any promises for when, but I will do my best.
Have you got time to give a look on this part? Or do you have any suggestions that I could help with?
I have looked closer at this, but haven't yet got the time to actually post a patch. Let me see if I can do some re-prioritization of things at my side...
Anyway, my plan is to post a "hackish" patch that we should use solely for testing, to make sure that the approach works. Then, I intend to continue to work on the complete solution, which seems to be requiring some reworks of locking parts in the clock framework. Although, it's too early for me, to say exactly how, so I can't really give you any pointers in this regard, sorry.
Quoted 7 lines
Thanks,
Peng.Quoted 3 lines
Kind regards
Uffe
Quoted 23 lines · Ulf Hansson
Quoted 9 lines · Peng Fan
Hi Ulf,
On 1/18/2023 8:55 PM, Ulf Hansson wrote:
...Quoted 4 lines
Quoted 3 lines
Honestly I not have idea on how to rework the clk part to avoid holding the clk-prepare lock when power on the phy and its power domain.
Quoted 8 lines
Quoted 5 lines
I understand. Let me try to prepare a hackish patch that we can use to run some tests.
Unfortunately, I can't make any promises for when, but I will do my best.
Have you got time to give a look on this part? Or do you have any suggestions that I could help with?
I have looked closer at this, but haven't yet got the time to actually post a patch. Let me see if I can do some re-prioritization of things at my side...
Quoted 6 lines
Anyway, my plan is to post a "hackish" patch that we should use solely for testing, to make sure that the approach works. Then, I intend to continue to work on the complete solution, which seems to be requiring some reworks of locking parts in the clock framework. Although, it's too early for me, to say exactly how, so I can't really give you any pointers in this regard, sorry.
I overlooked your reply, sorry. Thanks for still working on this. We have a new SoC has similar design that use blk ctrl which will need to work as clock controller, while it depends on power domain. So we will run into same deadlock issue. So I hope the deadlock issue could be resolved.
Thanks,
Peng.
Quoted 11 lines
Quoted 8 lines
Thanks,
Peng.Quoted 3 lines
Kind regards
Uffe