cpufreq: Upstream Android's Interactive governor

Hi Rafael et. al.,

We had some discussions [1] earlier on the PM mailing about upstreaming the cpufreq governor most widely used on Android Mobile phones and tablets: Interactive governor.

People (including Rafael) mostly agreed that we better get it upstreamed and here is an attempt to upstream most of it (idle notifiers aren't included in this series).

I picked the latest code spread over 70-80 patches from [2]. The unmodified code, based over mainline is pushed [4] for reference.

I have updated the governor to align it with the current practices followed with mainline governors, like using utilization hooks from the scheduler and handling kobject (for governor's sysfs directory) in a race free manner. And of course this included general cleanup of the governor as well. This version is pushed here [3] for testing.

The Android version of interactive governor also uses idle EXIT notifiers, but that code isn't part of this series and will be sent separately later. For people interested in looking at that, those are 3 minor patches on top of this series and are pushed here [5].

I haven't changed the core logic of the governor intentionally, as that rather requires more in-depth knowledge of the use case for which the optimizations have been done. So, we should do any such thing later on with new patches, so that people can fix regressions easily.

I also haven't tried to change the userspace interface as that may have broken the userspace that already exists and uses this governor.

This has been lightly tested on my Exynos board (dual ARM A15), where the governor gets inserted/removed multiple times, the sysfs files are all functional. The frequency gets changed with load, etc.

Amit Pundir (Linaro) has done extensive testing with Android and compared results of both Original Interactive governor and the modified one (in this series) and found them to be comparable.

This series is based of pm/bleeding-edge branch.

V1->V2:
- Changes to fix compilation issues with latest mainline
- Timer APIs got updated
- s/mod_timer_pinned/mod_timer
- s/init_timer/init_timer_pinned
- Updated prototypes of cpufreq_frequency_table_target() and update_util_handler()

--
viresh

[1] http://marc.info/?l=linux-pm&m=146301864519072
[2] https://android.googlesource.com/kernel/common remotes/android/android-4.4
[3] git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq/interactive
[4] git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq/interactive-orig
[5] git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq/intearctive-idle-notifier

Viresh Kumar (2):
cpufreq: Move gov_attr_* macros to cpufreq.h
cpufreq: Add android's 'interactive' governor

 Documentation/cpu-freq/governors.txt       |   86 ++
 drivers/cpufreq/Kconfig                    |   30 +
 drivers/cpufreq/Makefile                   |    1 +
 drivers/cpufreq/cpufreq_governor.h         |    8 -
 drivers/cpufreq/cpufreq_interactive.c      | 1363 ++++++++++++++++++++++++++++
 include/linux/cpufreq.h                    |   12 +
 include/trace/events/cpufreq_interactive.h |  112 +++
 kernel/sched/cpufreq_schedutil.c           |    8 +-
 8 files changed, 1608 insertions(+), 12 deletions(-)
 create mode 100644 drivers/cpufreq/cpufreq_interactive.c
 create mode 100644 include/trace/events/cpufreq_interactive.h

2.7.1.410.g6faf27b

These macros can be reused by governors which don't use the common governor code present in cpufreq_governor.c and should be moved to the relevant header.

Now that they are getting moved to the right header file, reuse them in schedutil governor as well (that required rename of show/store routines).

Also create gov_attr_wo() macro for write-only sysfs files, this will be used by Interactive governor in a later patch.

---
 drivers/cpufreq/cpufreq_governor.h |  8 --------
 include/linux/cpufreq.h            | 12 ++++++++++++
 kernel/sched/cpufreq_schedutil.c   |  8 ++++----
 3 files changed, 16 insertions(+), 12 deletions(-)

index ef1037e9c92b..3196ed9c1c0a 100644
@@ -70,14 +70,6 @@ static ssize_t show_##file_name						\
 	return sprintf(buf, "%u\n", dbs_data->file_name);		\
 }
 
-#define gov_attr_ro(_name)						\
-static struct governor_attr _name =					\
-__ATTR(_name, 0444, show_##_name, NULL)
-
-#define gov_attr_rw(_name)						\
-static struct governor_attr _name =					\
-__ATTR(_name, 0644, show_##_name, store_##_name)
-
 /* Common to all CPUs of a policy */
 struct policy_dbs_info {
 	struct cpufreq_policy *policy;

index 631ba33bbe9f..b65b1d795007 100644
@@ -539,6 +539,18 @@ struct gov_attr_set {
 	int usage_count;
 };
 
+#define gov_attr_ro(_name)						\
+static struct governor_attr _name =					\
+__ATTR(_name, 0444, show_##_name, NULL)
+
+#define gov_attr_wo(_name)						\
+static struct governor_attr _name =					\
+__ATTR(_name, 0200, NULL, store_##_name)
+
+#define gov_attr_rw(_name)						\
+static struct governor_attr _name =					\
+__ATTR(_name, 0644, show_##_name, store_##_name)
+
 /* sysfs ops for cpufreq governors */
 extern const struct sysfs_ops governor_sysfs_ops;
 

index 69e06898997d..c67c751944a9 100644
@@ -321,15 +321,15 @@ static inline struct sugov_tunables *to_sugov_tunables(struct gov_attr_set *attr
 	return container_of(attr_set, struct sugov_tunables, attr_set);
 }
 
-static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
+static ssize_t show_rate_limit_us(struct gov_attr_set *attr_set, char *buf)
 {
 	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
 
 	return sprintf(buf, "%u\n", tunables->rate_limit_us);
 }
 
-static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf,
-				   size_t count)
+static ssize_t store_rate_limit_us(struct gov_attr_set *attr_set,
+				   const char *buf, size_t count)
 {
 	struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
 	struct sugov_policy *sg_policy;

@@ -346,7 +346,7 @@ static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *bu
 	return count;
 }
 
-static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);
+gov_attr_rw(rate_limit_us);
 
 static struct attribute *sugov_attributes[] = {
 	&rate_limit_us.attr,

2.7.1.410.g6faf27b

Interactive governor has lived in Android sources for a very long time and this commit is based on the code present in following branch:

https://android.googlesource.com/kernel/common android-4.4

The Interactive governor is designed for latency-sensitive workloads, such as interactive user interfaces like the mobile phones and tablets. The interactive governor aims to be significantly more responsive to ramp CPU quickly up when CPU-intensive activity begins.

Existing governors sample CPU load at a particular rate, typically every X ms and then update the frequency from a work-handler. This can lead to under-powering UI threads for the period of time during which the user begins interacting with a previously-idle system until the next sample period happens.

The 'interactive' governor uses a different approach.

A real-time thread is used for scaling up, giving the remaining tasks the CPU performance benefit, unlike existing governors which are more likely to schedule ramp-up work to occur after your performance starved tasks have completed.

The Android version of interactive governor also checks whether to scale the CPU frequency up soon after coming out of idle. When the CPU comes out of idle, the governor check if the CPU sampling is overdue or not. If yes, it immediately starts the sampling. Otherwise, the utilization hooks from the scheduler handle the sampling later. If the CPU is very busy from exiting idle to when the evaluation happens, then it assumes that the CPU is under-powered and ramps it to MAX speed.

If the CPU was not sufficiently busy to immediately ramp to MAX speed, then the governor evaluates the CPU load since the last speed adjustment, choosing the highest value between that longer-term load or the short-term load since idle exit to determine the CPU speed to ramp to.

Idle notifiers will be be handled later and are not included for now.

The core of this code is written and maintained (in Android repositories) by Mike Chan and Todd Poyner over a long period of time.

Vireshk has made changes to to the governor to align it with the current practices followed with mainline governors, like using utilization hooks from the scheduler and handling kobject (for governor's sysfs directory) in a race free manner. And of course this included general cleanup of the governor as well.

---
 Documentation/cpu-freq/governors.txt       |   86 ++
 drivers/cpufreq/Kconfig                    |   30 +
 drivers/cpufreq/Makefile                   |    1 +
 drivers/cpufreq/cpufreq_interactive.c      | 1363 ++++++++++++++++++++++++++++
 include/trace/events/cpufreq_interactive.h |  112 +++
 5 files changed, 1592 insertions(+)
 create mode 100644 drivers/cpufreq/cpufreq_interactive.c
 create mode 100644 include/trace/events/cpufreq_interactive.h

index c15aa75f5227..0cf9a6bff6a5 100644
@@ -28,6 +28,7 @@
 2.3  Userspace
 2.4  Ondemand
 2.5  Conservative
+2.6  Interactive
 
 3.   The Governor Interface in the CPUfreq Core
 

@@ -218,6 +219,91 @@ a decision on when to decrease the frequency while running in any
 speed. Load for frequency increase is still evaluated every
 sampling rate.
 
+2.6 Interactive
+---------------
+
+The CPUfreq governor "interactive" is designed for latency-sensitive,
+interactive workloads. This governor sets the CPU speed depending on
+usage, similar to "ondemand" and "conservative" governors, but with a
+different set of configurable behaviors.
+
+The tunable values for this governor are:
+
+above_hispeed_delay: When speed is at or above hispeed_freq, wait for
+this long before raising speed in response to continued high load.
+The format is a single delay value, optionally followed by pairs of
+CPU speeds and the delay to use at or above those speeds.  Colons can
+be used between the speeds and associated delays for readability.  For
+example:
+
+   80000 1300000:200000 1500000:40000
+
+uses delay 80000 uS until CPU speed 1.3 GHz, at which speed delay
+200000 uS is used until speed 1.5 GHz, at which speed (and above)
+delay 40000 uS is used.  If speeds are specified these must appear in
+ascending order.  Default is 20000 uS.
+
+boost: If non-zero, immediately boost speed of all CPUs to at least
+hispeed_freq until zero is written to this attribute.  If zero, allow
+CPU speeds to drop below hispeed_freq according to load as usual.
+Default is zero.
+
+boostpulse: On each write, immediately boost speed of all CPUs to
+hispeed_freq for at least the period of time specified by
+boostpulse_duration, after which speeds are allowed to drop below
+hispeed_freq according to load as usual. Its a write-only file.
+
+boostpulse_duration: Length of time to hold CPU speed at hispeed_freq
+on a write to boostpulse, before allowing speed to drop according to
+load as usual.  Default is 80000 uS.
+
+go_hispeed_load: The CPU load at which to ramp to hispeed_freq.
+Default is 99%.
+
+hispeed_freq: An intermediate "high speed" at which to initially ramp
+when CPU load hits the value specified in go_hispeed_load.  If load
+stays high for the amount of time specified in above_hispeed_delay,
+then speed may be bumped higher.  Default is the maximum speed allowed
+by the policy at governor initialization time.
+
+io_is_busy: If set, the governor accounts IO time as CPU busy time.
+
+min_sample_time: The minimum amount of time to spend at the current
+frequency before ramping down. Default is 80000 uS.
+
+target_loads: CPU load values used to adjust speed to influence the
+current CPU load toward that value.  In general, the lower the target
+load, the more often the governor will raise CPU speeds to bring load
+below the target.  The format is a single target load, optionally
+followed by pairs of CPU speeds and CPU loads to target at or above
+those speeds.  Colons can be used between the speeds and associated
+target loads for readability.  For example:
+
+   85 1000000:90 1700000:99
+
+targets CPU load 85% below speed 1GHz, 90% at or above 1GHz, until
+1.7GHz and above, at which load 99% is targeted.  If speeds are
+specified these must appear in ascending order.  Higher target load
+values are typically specified for higher speeds, that is, target load
+values also usually appear in an ascending order. The default is
+target load 90% for all speeds.
+
+timer_rate: Sample rate for reevaluating CPU load when the CPU is not
+idle.  A deferrable timer is used, such that the CPU will not be woken
+from idle to service this timer until something else needs to run.
+(The maximum time to allow deferring this timer when not running at
+minimum speed is configurable via timer_slack.)  Default is 20000 uS.
+
+timer_slack: Maximum additional time to defer handling the governor
+sampling timer beyond timer_rate when running at speeds above the
+minimum.  For platforms that consume additional power at idle when
+CPUs are running at speeds greater than minimum, this places an upper
+bound on how long the timer will be deferred prior to re-evaluating
+load and dropping speed.  For example, if timer_rate is 20000uS and
+timer_slack is 10000uS then timers will be deferred for up to 30msec
+when not at lowest speed.  A value of -1 means defer timers
+indefinitely at all speeds.  Default is 80000 uS.
+
 3. The Governor Interface in the CPUfreq Core
 =============================================
 

index d8b164a7c4e5..d1c6a2fc4f26 100644
@@ -112,6 +112,16 @@ config CPU_FREQ_DEFAULT_GOV_SCHEDUTIL
 	  have a look at the help section of that governor. The fallback
 	  governor will be 'performance'.
 
+config CPU_FREQ_DEFAULT_GOV_INTERACTIVE
+	bool "interactive"
+	select CPU_FREQ_GOV_INTERACTIVE
+	select CPU_FREQ_GOV_PERFORMANCE
+	help
+	  Use the CPUFreq governor 'interactive' as default. This allows
+	  you to get a full dynamic cpu frequency capable system by simply
+	  loading your cpufreq low-level hardware driver, using the
+	  'interactive' governor for latency-sensitive workloads.
+
 endchoice
 
 config CPU_FREQ_GOV_PERFORMANCE

@@ -210,6 +220,26 @@ config CPU_FREQ_GOV_SCHEDUTIL
 
 	  If in doubt, say N.
 
+config CPU_FREQ_GOV_INTERACTIVE
+	tristate "'interactive' cpufreq policy governor"
+	depends on CPU_FREQ
+	select CPU_FREQ_GOV_ATTR_SET
+	select IRQ_WORK
+	help
+	  'interactive' - This driver adds a dynamic cpufreq policy governor
+	  designed for latency-sensitive workloads.
+
+	  This governor attempts to reduce the latency of clock
+	  increases so that the system is more responsive to
+	  interactive workloads.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called cpufreq_interactive.
+
+	  For details, take a look at linux/Documentation/cpu-freq.
+
+	  If in doubt, say N.
+
 comment "CPU frequency scaling drivers"
 
 config CPUFREQ_DT

index 0a9b6a093646..f0c9905d68a5 100644
@@ -10,6 +10,7 @@ obj-$(CONFIG_CPU_FREQ_GOV_POWERSAVE)	+= cpufreq_powersave.o
 obj-$(CONFIG_CPU_FREQ_GOV_USERSPACE)	+= cpufreq_userspace.o
 obj-$(CONFIG_CPU_FREQ_GOV_ONDEMAND)	+= cpufreq_ondemand.o
 obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE)	+= cpufreq_conservative.o
+obj-$(CONFIG_CPU_FREQ_GOV_INTERACTIVE)	+= cpufreq_interactive.o
 obj-$(CONFIG_CPU_FREQ_GOV_COMMON)		+= cpufreq_governor.o
 obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET)	+= cpufreq_governor_attr_set.o
 

2.7.1.410.g6faf27b