AMD Pstate Preferred Core

Drop CPU_SUP_INTE from SCHED_MC_PRIO, and it can be used by X86_AMD_PSTATE.

Signed-off-by Meng Li <[email protected]>

---
  arch/x86/Kconfig | 3 ++-
  1 file changed, 2 insertions(+), 1 deletion(-)

arch/x86/Kconfig +0 -0

index e36261b4ea14..03322d2840fa 100644

arch/x86/Kconfig +0 -0

  	select CPU_FREQ
  	default y
  	help

AMD Pstate driver utilizes the functions and data structures provided by the ITMT architecture to enable the scheduler to favor scheduling on cores which can be get a higher frequency with lower voltage. We call it AMD Pstate Preferrred Core.

Here sched_set_itmt_core_prio() is called to set priorities and sched_set_itmt_support() is called to enable ITMT feature. AMD Pstate driver uses the highest performance value to indicate the priority of CPU. The higher value has a higher priority.

The initial core rankings are set up by AMD Pstate when the system boots.

Add device attribute for preferred core states.

Add one new early parameter enable to allow user to

enable the preferred core if the processor and power firmware can support preferred core feature.

Signed-off-by Meng Li <[email protected]>

Signed-off-by Perry Yuan <[email protected]>

---
  drivers/cpufreq/amd-pstate.c | 120 ++++++++++++++++++++++++++++++-----
  1 file changed, 104 insertions(+), 16 deletions(-)

drivers/cpufreq/amd-pstate.c +104 -16

index 9a1e194d5cf8..d02305675f66 100644
@@ -37,6 +37,7 @@
  #include <linux/uaccess.h>
  #include <linux/static_call.h>
  #include <linux/amd-pstate.h>
+#include <linux/topology.h>
 
  #include <acpi/processor.h>
  #include <acpi/cppc_acpi.h>

@@ -49,6 +50,8 @@
 
  #define AMD_PSTATE_TRANSITION_LATENCY	20000
  #define AMD_PSTATE_TRANSITION_DELAY	1000
+#define AMD_PSTATE_PREFCORE_THRESHOLD	166
+#define AMD_PSTATE_MAX_CPPC_PERF	255
 
  /*
   * TODO: We need more time to fine tune processors with shared memory solution

@@ -65,6 +68,9 @@ static struct cpufreq_driver amd_pstate_epp_driver;
  static int cppc_state = AMD_PSTATE_UNDEFINED;
  static bool cppc_enabled;
 
+/*Preferred Core featue is supported*/
+static bool prefcore = true;
+
  /*
   * AMD Energy Preference Performance (EPP)
   * The EPP is used in the CCLK DPM controller to drive

@@ -290,23 +296,21 @@ static inline int amd_pstate_enable(bool enable)
  static int pstate_init_perf(struct amd_cpudata *cpudata)
  {
  	u64 cap1;
-	u32 highest_perf;
 
  	int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
  				     &cap1);
  	if (ret)
  		return ret;
 
-	/*
-	 * TODO: Introduce AMD specific power feature.
-	 *
-	 * CPPC entry doesn't indicate the highest performance in some ASICs.
+	/* For platforms that do not support the preferred core feature, the
+	 * highest_pef may be configured with 166 or 255, to avoid max frequency
+	 * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as
+	 * the default max perf.
  	 */
-	highest_perf = amd_get_highest_perf();
-	if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
-		highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
-
-	WRITE_ONCE(cpudata->highest_perf, highest_perf);
+	if (prefcore)
+		WRITE_ONCE(cpudata->highest_perf, AMD_PSTATE_PREFCORE_THRESHOLD);
+	else
+		WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));
 
  	WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
  	WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));

@@ -318,17 +322,15 @@ static int pstate_init_perf(struct amd_cpudata *cpudata)
  static int cppc_init_perf(struct amd_cpudata *cpudata)
  {
  	struct cppc_perf_caps cppc_perf;
-	u32 highest_perf;
 
  	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
  	if (ret)
  		return ret;
 
-	highest_perf = amd_get_highest_perf();
-	if (highest_perf > cppc_perf.highest_perf)
-		highest_perf = cppc_perf.highest_perf;
-
-	WRITE_ONCE(cpudata->highest_perf, highest_perf);
+	if (prefcore)
+		WRITE_ONCE(cpudata->highest_perf, AMD_PSTATE_PREFCORE_THRESHOLD);
+	else
+		WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf);
 
  	WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
  	WRITE_ONCE(cpudata->lowest_nonlinear_perf,

@@ -676,6 +678,72 @@ static void amd_perf_ctl_reset(unsigned int cpu)
  	wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
  }
 
+/*
+ * Set AMD Pstate Preferred Core enable can't be done directly from cpufreq callbacks
+ * due to locking, so queue the work for later.
+ */
+static void amd_pstste_sched_prefcore_workfn(struct work_struct *work)
+{
+	sched_set_itmt_support();
+}
+static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn);
+
+/**
+ * Get the highest performance register value.
+ * @cpu: CPU from which to get highest performance.
+ * @highest_perf: Return address.
+ *
+ * Return: 0 for success, -EIO otherwise.
+ */
+static int amd_pstate_get_highest_perf(int cpu, u64 *highest_perf)
+{
+       int ret;
+
+       if (boot_cpu_has(X86_FEATURE_CPPC)) {
+               u64 cap1;
+
+               ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1);
+               if (ret)
+                       return ret;
+               WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));
+       } else {
+               ret = cppc_get_highest_perf(cpu, highest_perf);
+       }
+
+       return (ret);
+}
+
+static void amd_pstate_init_prefcore(void)
+{
+	int cpu, ret;
+	u64 highest_perf;
+
+	if (!prefcore)
+		return;
+
+	for_each_online_cpu(cpu) {
+		ret = amd_pstate_get_highest_perf(cpu, &highest_perf);
+		if (ret)
+			break;
+
+		sched_set_itmt_core_prio(highest_perf, cpu);
+
+		/* check if CPPC preferred core feature is enabled*/
+		if (highest_perf == AMD_PSTATE_MAX_CPPC_PERF) {
+			prefcore = false;
+			return;
+		}
+	}
+
+	/*
+	 * This code can be run during CPU online under the
+	 * CPU hotplug locks, so sched_set_amd_prefcore_support()
+	 * cannot be called from here.  Queue up a work item
+	 * to invoke it.
+	 */
+	schedule_work(&sched_prefcore_work);
+}
+
  static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
  {
  	int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;

@@ -1037,6 +1105,12 @@ static ssize_t status_store(struct device *a, struct device_attribute *b,
  	return ret < 0 ? ret : count;
  }
 
+static ssize_t prefcore_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%s\n", prefcore ? "enabled" : "disabled");
+}
+
  cpufreq_freq_attr_ro(amd_pstate_max_freq);
  cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
 

@@ -1044,6 +1118,7 @@ cpufreq_freq_attr_ro(amd_pstate_highest_perf);
  cpufreq_freq_attr_rw(energy_performance_preference);
  cpufreq_freq_attr_ro(energy_performance_available_preferences);
  static DEVICE_ATTR_RW(status);
+static DEVICE_ATTR_RO(prefcore);
 
  static struct freq_attr *amd_pstate_attr[] = {
  	&amd_pstate_max_freq,

@@ -1063,6 +1138,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = {
 
  static struct attribute *pstate_global_attributes[] = {
  	&dev_attr_status.attr,
+	&dev_attr_prefcore.attr,
  	NULL
  };
 

@@ -1506,6 +1582,8 @@ static int __init amd_pstate_init(void)
  		}
  	}
 
+	amd_pstate_init_prefcore();
+
  	return ret;
 
  global_attr_free:

@@ -1527,7 +1605,17 @@ static int __init amd_pstate_param(char *str)
 
  	return amd_pstate_set_driver(mode_idx);
  }
+
+static int __init amd_prefcore_param(char *str)
+{
+	if (!strcmp(str, "disable"))
+		prefcore = false;
+
+	return 0;
+}
+
  early_param("amd_pstate", amd_pstate_param);
+early_param("amd_prefcore", amd_prefcore_param);
 
  MODULE_AUTHOR("Huang Rui <[email protected]>");
  MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");

Please refer to the ACPI_Spec for details on the highest performance and notify events of CPPC.

Signed-off-by Meng Li <[email protected]>

Link https://uefi.org/specs/ACPI/6.5/08_Processor_Configuration_and_Control.html?highlight=cppc#cpc-continuous-performance-control

---
  drivers/acpi/processor_driver.c |  6 ++++++
  drivers/cpufreq/cpufreq.c       | 13 +++++++++++++
  include/linux/cpufreq.h         |  4 ++++
  3 files changed, 23 insertions(+)

drivers/acpi/processor_driver.c +6 -0

index 4bd16b3f0781..29b2fb68a35d 100644
@@ -27,6 +27,7 @@
  #define ACPI_PROCESSOR_NOTIFY_PERFORMANCE 0x80
  #define ACPI_PROCESSOR_NOTIFY_POWER	0x81
  #define ACPI_PROCESSOR_NOTIFY_THROTTLING	0x82
+#define ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED	0x85
 
  MODULE_AUTHOR("Paul Diefenbaugh");
  MODULE_DESCRIPTION("ACPI Processor Driver");

@@ -83,6 +84,11 @@ static void acpi_processor_notify(acpi_handle handle, u32 event, void *data)
  		acpi_bus_generate_netlink_event(device->pnp.device_class,
  						  dev_name(&device->dev), event, 0);
  		break;
+	case ACPI_PROCESSOR_NOTIFY_HIGEST_PERF_CHANGED:
+		cpufreq_update_highest_perf(pr->id);
+		acpi_bus_generate_netlink_event(device->pnp.device_class,
+						  dev_name(&device->dev), event, 0);
+		break;
  	default:
  		acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event);
  		break;

drivers/cpufreq/cpufreq.c +13 -0

index 50bbc969ffe5..842357abfae6 100644
@@ -2675,6 +2675,19 @@ void cpufreq_update_limits(unsigned int cpu)
  }
  EXPORT_SYMBOL_GPL(cpufreq_update_limits);
 
+/**
+ * cpufreq_update_highest_perf - Update highest performance for a given CPU.
+ * @cpu: CPU to update the highest performance for.
+ *
+ * Invoke the driver's ->update_highest_perf callback if present
+ */
+void cpufreq_update_highest_perf(unsigned int cpu)
+{
+	if (cpufreq_driver->update_highest_perf)
+		cpufreq_driver->update_highest_perf(cpu);
+}
+EXPORT_SYMBOL_GPL(cpufreq_update_highest_perf);
+
  /*********************************************************************
   *               BOOST						     *
   *********************************************************************/

include/linux/cpufreq.h +4 -0

index 172ff51c1b2a..766c83a4fae7 100644
@@ -231,6 +231,7 @@ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu);
  void refresh_frequency_limits(struct cpufreq_policy *policy);
  void cpufreq_update_policy(unsigned int cpu);
  void cpufreq_update_limits(unsigned int cpu);
+void cpufreq_update_highest_perf(unsigned int cpu);
  bool have_governor_per_policy(void);
  bool cpufreq_supports_freq_invariance(void);
  struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy);

@@ -376,6 +377,9 @@ struct cpufreq_driver {
  	/* Called to update policy limits on firmware notifications. */
  	void		(*update_limits)(unsigned int cpu);
 
+	/* Called to update highest performance on firmware notifications. */
+	void		(*update_highest_perf)(unsigned int cpu);
+
  	/* optional */
  	int		(*bios_limit)(int cpu, unsigned int *limit);
 

Introduce AMD Pstate Preferred Core.

check preferred core state:

$ cat /sys/devices/system/cpu/amd-pstate/prefcore_state

Signed-off-by Meng Li <[email protected]>

---
  Documentation/admin-guide/pm/amd-pstate.rst | 54 +++++++++++++++++++++
  1 file changed, 54 insertions(+)

Documentation/admin-guide/pm/amd-pstate.rst +30 -0

index 1cf40f69278c..ef2b69935311 100644
@@ -353,6 +353,48 @@ is activated.  In this mode, driver requests minimum and maximum performance
  level and the platform autonomously selects a performance level in this range
  and appropriate to the current workload.
 
+AMD Pstate Preferred Core
+=================================
+
+The core frequency is subjected to the process variation in semiconductors.
+Not all cores are able to reach the maximum frequency respecting the
+infrastructure limits. Consequently, AMD has redefined the concept of
+maximum frequency of a part. This means that a fraction of cores can reach
+maximum frequency. To find the best process scheduling policy for a given
+scenario, OS needs to know the core ordering informed by the platform through
+highest performance capability register of the CPPC interface.
+
+``AMD Pstate Preferred Core`` enable the scheduler to favor scheduling on cores
+can be get a higher frequency with lower voltage under preferred core.
+And it has the ability to dynamically change the preferred core based on the
+workload and platform conditions and accounting for thermals and aging.
+
+The priority metric will be initialized by the AMD Pstate driver. The AMD Pstate
+driver will also determine whether or not ``AMD Pstate Preferred Core`` is
+supported by the platform.
+
+AMD Pstate driver will provide an initial core ordering when the system boots.
+The platform uses the CPPC interfaces to communicate the core ranking to the
+operating system and scheduler to make sure that OS is choosing the cores
+with highest performance firstly for scheduling the process. When AMD Pstate
+driver receives a message with the highest performance change, it will
+update the core ranking and set the cpu's priority.
+
+AMD Preferred Core Switch
+=================================
+Kernel Parameters

Documentation/admin-guide/pm/amd-pstate.rst +13 -0

+
  User Space Interface in ``sysfs`` - General
  ===========================================
 

@@ -385,6 +427,18 @@ control its functionality at the system level.  They are located in the
          to the operation mode represented by that string - or to be
          unregistered in the "disable" case.
 
+``prefcore``
+	Preferred Core state of the driver: "enabled" or "disabled".
+
+	"enabled"
+		Enable the AMD Preferred Core.
+
+	"disabled"
+		Disable the AMD Preferred Core
+
+
+        This attribute is read-only to check the state of Preferred Core.
+
  ``cpupower`` tool support for ``amd-pstate``
  ===============================================
 

AMD Pstate driver support enable/disable Preferred core. Default enabled on platforms supporting AMD Preferred Core. Disable AMD Pstate Preferred Core with "amd_prefcore=disable" added to the kernel command line.

Signed-off-by Meng Li <[email protected]>

---
  Documentation/admin-guide/kernel-parameters.txt | 5 +++++
  1 file changed, 5 insertions(+)

Documentation/admin-guide/kernel-parameters.txt +5 -0

index 23ebe34ff901..4f78067bb8af 100644
@@ -363,6 +363,11 @@
  			  selects a performance level in this range and appropriate
  			  to the current workload.
 
+	amd_prefcore=
+			[X86]
+			disable
+			  Disable AMD Pstate Preferred Core.
+
  	amijoy.map=	[HW,JOY] Amiga joystick support
  			Map of devices attached to JOY0DAT and JOY1DAT
  			Format: <a>,<b>