Skip to contentSkip to conversation

x86/resctrl: Fix arch_mbm_* array overrun on SNC

PATCH Superseded Peter Newman 2 participants 3 messages Jul 22, 2024 – Aug 22, 2024

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86/resctrl: Fix arch_mbm_* array overrun on SNC

When using resctrl on systems with Sub-NUMA Clustering enabled, monitoring groups may be allocated RMID values which would overrun the arch_mbm_{local,total} arrays.

This is due to inconsistencies in whether the SNC-adjusted num_rmid value or the unadjusted value in resctrl_arch_system_num_rmid_idx() is used. The num_rmid value for the L3 resource is currently:

resctrl_arch_system_num_rmid_idx() / snc_nodes_per_l3_cache

As a simple fix, make resctrl_arch_system_num_rmid_idx() return the SNC-adjusted, L3 num_rmid value on x86.

Fixes e13db55b5a0d ("x86/resctrl: Introduce snc_nodes_per_l3_cache")
---
 arch/x86/include/asm/resctrl.h     | 6 ------
 arch/x86/kernel/cpu/resctrl/core.c | 8 ++++++++
 include/linux/resctrl.h            | 3 +++
 3 files changed, 11 insertions(+), 6 deletions(-)
arch/x86/include/asm/resctrl.h +0 -6
index 12dbd2588ca7..8b1b6ce1e51b 100644
@@ -156,12 +156,6 @@ static inline void resctrl_sched_in(struct task_struct *tsk)
 		__resctrl_sched_in(tsk);
 }
 
-static inline u32 resctrl_arch_system_num_rmid_idx(void)
-{
-	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
-	return boot_cpu_data.x86_cache_max_rmid + 1;
-}
-
 static inline void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid)
 {
 	*rmid = idx;
arch/x86/kernel/cpu/resctrl/core.c +8 -0
index 1930fce9dfe9..8591d53c144b 100644
@@ -119,6 +119,14 @@ struct rdt_hw_resource rdt_resources_all[] = {
 	},
 };
 
+u32 resctrl_arch_system_num_rmid_idx(void)
+{
+	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+
+	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
+	return r->num_rmid;
+}
+
 /*
  * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs
  * as they do not have CPUID enumeration support for Cache allocation.
include/linux/resctrl.h +3 -0
index b0875b99e811..43ac241471b3 100644
@@ -248,6 +248,9 @@ struct resctrl_schema {
 
 /* The number of closid supported by this resource regardless of CDP */
 u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
+
+u32 resctrl_arch_system_num_rmid_idx(void);
+
 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid);
 
 /*
Signature · 1 line
2.45.2.1089.g2a221341d9-goog
maintainer
#1 lore raw

Hi Peter,

Quoted 4 lines · Peter Newman

When using resctrl on systems with Sub-NUMA Clustering enabled, monitoring groups may be allocated RMID values which would overrun the arch_mbm_{local,total} arrays.

Quoted 9 lines

This is due to inconsistencies in whether the SNC-adjusted num_rmid value or the unadjusted value in resctrl_arch_system_num_rmid_idx() is used. The num_rmid value for the L3 resource is currently:

resctrl_arch_system_num_rmid_idx() / snc_nodes_per_l3_cache

As a simple fix, make resctrl_arch_system_num_rmid_idx() return the SNC-adjusted, L3 num_rmid value on x86.

Thank you very much for finding, root-causing, and providing a fix for the issue.

Quoted 49 lines
Fixes e13db55b5a0d ("x86/resctrl: Introduce snc_nodes_per_l3_cache")
---
  arch/x86/include/asm/resctrl.h     | 6 ------
  arch/x86/kernel/cpu/resctrl/core.c | 8 ++++++++
  include/linux/resctrl.h            | 3 +++
  3 files changed, 11 insertions(+), 6 deletions(-)
arch/x86/include/asm/resctrl.h +0 -6
index 12dbd2588ca7..8b1b6ce1e51b 100644
@@ -156,12 +156,6 @@ static inline void resctrl_sched_in(struct task_struct *tsk)
  		__resctrl_sched_in(tsk);
  }
 
-static inline u32 resctrl_arch_system_num_rmid_idx(void)
-{
-	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
-	return boot_cpu_data.x86_cache_max_rmid + 1;
-}
-
  static inline void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid)
  {
  	*rmid = idx;
arch/x86/kernel/cpu/resctrl/core.c +8 -0
index 1930fce9dfe9..8591d53c144b 100644
@@ -119,6 +119,14 @@ struct rdt_hw_resource rdt_resources_all[] = {
  	},
  };
 
+u32 resctrl_arch_system_num_rmid_idx(void)
+{
+	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+
+	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
+	return r->num_rmid;
+}
+
  /*
   * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs
   * as they do not have CPUID enumeration support for Cache allocation.
include/linux/resctrl.h +0 -0
index b0875b99e811..43ac241471b3 100644

include/linux/resctrl.h struct resctrl_schema { · line 253

Quoted 6 lines
include/linux/resctrl.h +3 -0
@@ -248,6 +248,9 @@ struct resctrl_schema {
 
  /* The number of closid supported by this resource regardless of CDP */
  u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
+
+u32 resctrl_arch_system_num_rmid_idx(void);
+

nit: the additional empty lines are unnecessary.

include/linux/resctrl.h struct resctrl_schema { · line 256

Quoted 3 lines
include/linux/resctrl.h +0 -0
  int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid);
 
  /*

| Reviewed-by: Reinette Chatre <[email protected]>

Reinette

maintainer
#2 lore raw

Hi Peter,

Quoted 73 lines · Reinette Chatre
Quoted 4 lines · Peter Newman

When using resctrl on systems with Sub-NUMA Clustering enabled, monitoring groups may be allocated RMID values which would overrun the arch_mbm_{local,total} arrays.

Quoted 9 lines

This is due to inconsistencies in whether the SNC-adjusted num_rmid value or the unadjusted value in resctrl_arch_system_num_rmid_idx() is used. The num_rmid value for the L3 resource is currently:

  resctrl_arch_system_num_rmid_idx() / snc_nodes_per_l3_cache

As a simple fix, make resctrl_arch_system_num_rmid_idx() return the SNC-adjusted, L3 num_rmid value on x86.

Thank you very much for finding, root-causing, and providing a fix for the issue.

Quoted 47 lines
Fixes e13db55b5a0d ("x86/resctrl: Introduce snc_nodes_per_l3_cache")

---
  arch/x86/include/asm/resctrl.h     | 6 ------
  arch/x86/kernel/cpu/resctrl/core.c | 8 ++++++++
  include/linux/resctrl.h            | 3 +++

  3 files changed, 11 insertions(+), 6 deletions(-)
arch/x86/include/asm/resctrl.h +0 -6
index 12dbd2588ca7..8b1b6ce1e51b 100644
@@ -156,12 +156,6 @@ static inline void resctrl_sched_in(struct task_struct *tsk)
           __resctrl_sched_in(tsk);
   }
-static inline u32 resctrl_arch_system_num_rmid_idx(void)
-{
-    /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
-    return boot_cpu_data.x86_cache_max_rmid + 1;
-}
-
   static inline void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid)
   {
       *rmid = idx;
arch/x86/kernel/cpu/resctrl/core.c +8 -0
index 1930fce9dfe9..8591d53c144b 100644
@@ -119,6 +119,14 @@ struct rdt_hw_resource rdt_resources_all[] = {
       },
   };
+u32 resctrl_arch_system_num_rmid_idx(void)
+{
+    struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+
+    /* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
+    return r->num_rmid;
+}
+
   /*
    * cache_alloc_hsw_probe() - Have to probe for Intel haswell server CPUs
    * as they do not have CPUID enumeration support for Cache allocation.
include/linux/resctrl.h +0 -0
index b0875b99e811..43ac241471b3 100644

include/linux/resctrl.h struct resctrl_schema { · line 252

Quoted 5 lines
include/linux/resctrl.h +3 -0
@@ -248,6 +248,9 @@ struct resctrl_schema {
   /* The number of closid supported by this resource regardless of CDP */
   u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
+
+u32 resctrl_arch_system_num_rmid_idx(void);
+

nit: the additional empty lines are unnecessary.

Quoted 5 lines

include/linux/resctrl.h struct resctrl_schema { · line 256

Quoted 2 lines
include/linux/resctrl.h +0 -0
   int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid);
   /*

| Reviewed-by: Reinette Chatre <[email protected]>

It would be great if this fix can be included in the kernel release. Could you please send V2 with nit and tag applied to be ready for inclusion?

Thank you

Reinette