Skip to contentSkip to conversation

x86/resctrl: monitored closid+rmid together, separate arch/fs locking

PATCH Applied James Morse 8 participants 104 messages Feb 13, 2024 – Apr 24, 2024 Page 2 of 2

Subsystems (3)
lore
Share & tools
lore
merge bot Applied
#92 lore raw

[tip: x86/cache] x86/resctrl: Pass domain to target CPU

In reply to Tony Luck
The following commit has been merged into the x86/cache branch of tip:

The following commit has been merged into the x86/cache branch of tip:

Commit-ID:     e3ca96e479c91d6ee657d3caa5092a6a3a620f9f
Gitweb:        https://git.kernel.org/tip/e3ca96e479c91d6ee657d3caa5092a6a3a620f9f
Author:        Tony Luck <[email protected]>
AuthorDate:    Fri, 08 Mar 2024 13:38:45 -08:00
Committer:     Borislav Petkov (AMD) <[email protected]>
CommitterDate: Wed, 24 Apr 2024 13:41:41 +02:00

x86/resctrl: Pass domain to target CPU

reset_all_ctrls() and resctrl_arch_update_domains() use on_each_cpu_mask() to call rdt_ctrl_update() on potentially one CPU from each domain.

But this means rdt_ctrl_update() needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely if cpus_lock is held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.

Commit

c0d848fcb09d ("x86/resctrl: Remove lockdep annotation that triggers false positive")

removed the incorrect assertions.

Add the target domain to the msr_param structure and call rdt_ctrl_update() for each domain separately using smp_call_function_single(). This means that rdt_ctrl_update() doesn't need to search for the domain and get_domain_from_cpu() can safely assert that the cpus_lock is held since the remaining callers do not use IPI.

---
 arch/x86/kernel/cpu/resctrl/core.c        | 17 +++-------
 arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 38 ++++------------------
 arch/x86/kernel/cpu/resctrl/internal.h    |  2 +-
 arch/x86/kernel/cpu/resctrl/rdtgroup.c    | 12 +------
 4 files changed, 17 insertions(+), 52 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +6 -11
index 83e4034..acf52aa 100644
@@ -362,6 +362,8 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)
 {
 	struct rdt_domain *d;
 
+	lockdep_assert_cpus_held();
+
 	list_for_each_entry(d, &r->domains, list) {
 		/* Find the domain that contains this CPU */
 		if (cpumask_test_cpu(cpu, &d->cpu_mask))
@@ -378,19 +380,11 @@ u32 resctrl_arch_get_num_closid(struct rdt_resource *r)
 
 void rdt_ctrl_update(void *arg)
 {
+	struct rdt_hw_resource *hw_res;
 	struct msr_param *m = arg;
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
-	struct rdt_resource *r = m->res;
-	int cpu = smp_processor_id();
-	struct rdt_domain *d;
 
-	d = get_domain_from_cpu(cpu, r);
-	if (d) {
-		hw_res->msr_update(d, m, r);
-		return;
-	}
-	pr_warn_once("cpu %d not found in any domain for resource %s\n",
-		     cpu, r->name);
+	hw_res = resctrl_to_arch_res(m->res);
+	hw_res->msr_update(m->dom, m, m->res);
 }
 
 /*
@@ -463,6 +457,7 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
 	hw_dom->ctrl_val = dc;
 	setup_default_ctrlval(r, dc);
 
+	m.dom = d;
 	m.low = 0;
 	m.high = hw_res->num_closid;
 	hw_res->msr_update(d, &m, r);
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +7 -31
index 7997b47..165d8d4 100644
@@ -272,22 +272,6 @@ static u32 get_config_index(u32 closid, enum resctrl_conf_type type)
 	}
 }
 
-static bool apply_config(struct rdt_hw_domain *hw_dom,
-			 struct resctrl_staged_config *cfg, u32 idx,
-			 cpumask_var_t cpu_mask)
-{
-	struct rdt_domain *dom = &hw_dom->d_resctrl;
-
-	if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) {
-		cpumask_set_cpu(cpumask_any(&dom->cpu_mask), cpu_mask);
-		hw_dom->ctrl_val[idx] = cfg->new_ctrl;
-
-		return true;
-	}
-
-	return false;
-}
-
 int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
 			    u32 closid, enum resctrl_conf_type t, u32 cfg_val)
 {
@@ -302,6 +286,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
 	hw_dom->ctrl_val[idx] = cfg_val;
 
 	msr_param.res = r;
+	msr_param.dom = d;
 	msr_param.low = idx;
 	msr_param.high = idx + 1;
 	hw_res->msr_update(d, &msr_param, r);
@@ -315,48 +300,39 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 	struct rdt_hw_domain *hw_dom;
 	struct msr_param msr_param;
 	enum resctrl_conf_type t;
-	cpumask_var_t cpu_mask;
 	struct rdt_domain *d;
 	u32 idx;
 
 	/* Walking r->domains, ensure it can't race with cpuhp */
 	lockdep_assert_cpus_held();
 
-	if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
-		return -ENOMEM;
-
-	msr_param.res = NULL;
 	list_for_each_entry(d, &r->domains, list) {
 		hw_dom = resctrl_to_arch_dom(d);
+		msr_param.res = NULL;
 		for (t = 0; t < CDP_NUM_TYPES; t++) {
 			cfg = &hw_dom->d_resctrl.staged_config[t];
 			if (!cfg->have_new_ctrl)
 				continue;
 
 			idx = get_config_index(closid, t);
-			if (!apply_config(hw_dom, cfg, idx, cpu_mask))
+			if (cfg->new_ctrl == hw_dom->ctrl_val[idx])
 				continue;
+			hw_dom->ctrl_val[idx] = cfg->new_ctrl;
 
 			if (!msr_param.res) {
 				msr_param.low = idx;
 				msr_param.high = msr_param.low + 1;
 				msr_param.res = r;
+				msr_param.dom = d;
 			} else {
 				msr_param.low = min(msr_param.low, idx);
 				msr_param.high = max(msr_param.high, idx + 1);
 			}
 		}
+		if (msr_param.res)
+			smp_call_function_any(&d->cpu_mask, rdt_ctrl_update, &msr_param, 1);
 	}
 
-	if (cpumask_empty(cpu_mask))
-		goto done;
-
-	/* Update resource control msr on all the CPUs. */
-	on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1);
-
-done:
-	free_cpumask_var(cpu_mask);
-
 	return 0;
 }
 
arch/x86/kernel/cpu/resctrl/internal.h +2 -0
index 1a8687f..ab2d315 100644
@@ -379,11 +379,13 @@ static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r)
 /**
  * struct msr_param - set a range of MSRs from a domain
  * @res:       The resource to use
+ * @dom:       The domain to update
  * @low:       Beginning index from base MSR
  * @high:      End index
  */
 struct msr_param {
 	struct rdt_resource	*res;
+	struct rdt_domain	*dom;
 	u32			low;
 	u32			high;
 };
arch/x86/kernel/cpu/resctrl/rdtgroup.c +2 -10
index 011e17e..02f213f 100644
@@ -2813,16 +2813,12 @@ static int reset_all_ctrls(struct rdt_resource *r)
 	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 	struct rdt_hw_domain *hw_dom;
 	struct msr_param msr_param;
-	cpumask_var_t cpu_mask;
 	struct rdt_domain *d;
 	int i;
 
 	/* Walking r->domains, ensure it can't race with cpuhp */
 	lockdep_assert_cpus_held();
 
-	if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
-		return -ENOMEM;
-
 	msr_param.res = r;
 	msr_param.low = 0;
 	msr_param.high = hw_res->num_closid;
@@ -2834,17 +2830,13 @@ static int reset_all_ctrls(struct rdt_resource *r)
 	 */
 	list_for_each_entry(d, &r->domains, list) {
 		hw_dom = resctrl_to_arch_dom(d);
-		cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
 
 		for (i = 0; i < hw_res->num_closid; i++)
 			hw_dom->ctrl_val[i] = r->default_ctrl;
+		msr_param.dom = d;
+		smp_call_function_any(&d->cpu_mask, rdt_ctrl_update, &msr_param, 1);
 	}
 
-	/* Update CBM on all the CPUs in cpu_mask */
-	on_each_cpu_mask(cpu_mask, rdt_ctrl_update, &msr_param, 1);
-
-	free_cpumask_var(cpu_mask);
-
 	return 0;
 }
 
maintainer
#93 lore raw

PATCH v5 2/2 x86/resctrl: Simplify call convention for MSR update functions

In reply to Tony Luck

The per-resource MSR update functions cat_wrmsr(), mba_wrmsr_intel(), and mba_wrmsr_amd() all take three arguments:

(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)

struct msr_param contains pointers to both struct rdt_resource and struct rdt_domain, thus only struct msr_param is necessary.

Pass struct msr_param as a single parameter. Clean up formatting and fix some fir tree declaration ordering.

No functional change.

---
 arch/x86/kernel/cpu/resctrl/internal.h    |  3 +-
 arch/x86/kernel/cpu/resctrl/core.c        | 40 +++++++++--------------
 arch/x86/kernel/cpu/resctrl/ctrlmondata.c |  2 +-
 3 files changed, 18 insertions(+), 27 deletions(-)
arch/x86/kernel/cpu/resctrl/internal.h +1 -2
index bc999471f072..8f40fb35db78 100644
@@ -444,8 +444,7 @@ struct rdt_hw_resource {
 	struct rdt_resource	r_resctrl;
 	u32			num_closid;
 	unsigned int		msr_base;
-	void (*msr_update)	(struct rdt_domain *d, struct msr_param *m,
-				 struct rdt_resource *r);
+	void			(*msr_update)(struct msr_param *m);
 	unsigned int		mon_scale;
 	unsigned int		mbm_width;
 	unsigned int		mbm_cfg_mask;
arch/x86/kernel/cpu/resctrl/core.c +16 -24
index acf52aa185e0..7751eea19fd2 100644
@@ -56,14 +56,9 @@ int max_name_width, max_data_width;
  */
 bool rdt_alloc_capable;
 
-static void
-mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m,
-		struct rdt_resource *r);
-static void
-cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r);
-static void
-mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m,
-	      struct rdt_resource *r);
+static void mba_wrmsr_intel(struct msr_param *m);
+static void cat_wrmsr(struct msr_param *m);
+static void mba_wrmsr_amd(struct msr_param *m);
 
 #define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)
 
@@ -309,12 +304,11 @@ static void rdt_get_cdp_l2_config(void)
 	rdt_get_cdp_config(RDT_RESOURCE_L2);
 }
 
-static void
-mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
+static void mba_wrmsr_amd(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	for (i = m->low; i < m->high; i++)
 		wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -334,25 +328,22 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
 	return r->default_ctrl;
 }
 
-static void
-mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m,
-		struct rdt_resource *r)
+static void mba_wrmsr_intel(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	/*  Write the delay values for mba. */
 	for (i = m->low; i < m->high; i++)
-		wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], r));
+		wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res));
 }
 
-static void
-cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
+static void cat_wrmsr(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	for (i = m->low; i < m->high; i++)
 		wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -384,7 +375,7 @@ void rdt_ctrl_update(void *arg)
 	struct msr_param *m = arg;
 
 	hw_res = resctrl_to_arch_res(m->res);
-	hw_res->msr_update(m->dom, m, m->res);
+	hw_res->msr_update(m);
 }
 
 /*
@@ -457,10 +448,11 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
 	hw_dom->ctrl_val = dc;
 	setup_default_ctrlval(r, dc);
 
+	m.res = r;
 	m.dom = d;
 	m.low = 0;
 	m.high = hw_res->num_closid;
-	hw_res->msr_update(d, &m, r);
+	hw_res->msr_update(&m);
 	return 0;
 }
 
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
index 165d8d453c04..b7291f60399c 100644
@@ -289,7 +289,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
 	msr_param.dom = d;
 	msr_param.low = idx;
 	msr_param.high = idx + 1;
-	hw_res->msr_update(d, &msr_param, r);
+	hw_res->msr_update(&msr_param);
 
 	return 0;
 }
Signature · 1 line
2.43.0
reviewer Reviewed-by on v5 2/2
#94 lore raw

Re: [PATCH v5 2/2] x86/resctrl: Simplify call convention for MSR update functions

Quoted 11 lines · Tony Luck

The per-resource MSR update functions cat_wrmsr(), mba_wrmsr_intel(), and mba_wrmsr_amd() all take three arguments:

(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)

struct msr_param contains pointers to both struct rdt_resource and struct rdt_domain, thus only struct msr_param is necessary.

Pass struct msr_param as a single parameter. Clean up formatting and fix some fir tree declaration ordering.

Quoted 6 lines

No functional change.

merge bot Applied
#95 lore raw

[tip: x86/cache] x86/resctrl: Simplify call convention for MSR update functions

In reply to Tony Luck
The following commit has been merged into the x86/cache branch of tip:

The following commit has been merged into the x86/cache branch of tip:

Commit-ID:     bd4955d4bc2182ccb660c9c30a4dd7f36feaf943
Gitweb:        https://git.kernel.org/tip/bd4955d4bc2182ccb660c9c30a4dd7f36feaf943
Author:        Tony Luck <[email protected]>
AuthorDate:    Fri, 08 Mar 2024 13:38:46 -08:00
Committer:     Borislav Petkov (AMD) <[email protected]>
CommitterDate: Wed, 24 Apr 2024 13:47:00 +02:00

x86/resctrl: Simplify call convention for MSR update functions

The per-resource MSR update functions cat_wrmsr(), mba_wrmsr_intel(), and mba_wrmsr_amd() all take three arguments:

(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)

struct msr_param contains pointers to both struct rdt_resource and struct rdt_domain, thus only struct msr_param is necessary.

Pass struct msr_param as a single parameter. Clean up formatting and fix some fir tree declaration ordering.

No functional change.

---
 arch/x86/kernel/cpu/resctrl/core.c        | 40 ++++++++--------------
 arch/x86/kernel/cpu/resctrl/ctrlmondata.c |  2 +-
 arch/x86/kernel/cpu/resctrl/internal.h    |  3 +--
 3 files changed, 18 insertions(+), 27 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +16 -24
index acf52aa..7751eea 100644
@@ -56,14 +56,9 @@ int max_name_width, max_data_width;
  */
 bool rdt_alloc_capable;
 
-static void
-mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m,
-		struct rdt_resource *r);
-static void
-cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r);
-static void
-mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m,
-	      struct rdt_resource *r);
+static void mba_wrmsr_intel(struct msr_param *m);
+static void cat_wrmsr(struct msr_param *m);
+static void mba_wrmsr_amd(struct msr_param *m);
 
 #define domain_init(id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)
 
@@ -309,12 +304,11 @@ static void rdt_get_cdp_l2_config(void)
 	rdt_get_cdp_config(RDT_RESOURCE_L2);
 }
 
-static void
-mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
+static void mba_wrmsr_amd(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	for (i = m->low; i < m->high; i++)
 		wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -334,25 +328,22 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
 	return r->default_ctrl;
 }
 
-static void
-mba_wrmsr_intel(struct rdt_domain *d, struct msr_param *m,
-		struct rdt_resource *r)
+static void mba_wrmsr_intel(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	/*  Write the delay values for mba. */
 	for (i = m->low; i < m->high; i++)
-		wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], r));
+		wrmsrl(hw_res->msr_base + i, delay_bw_map(hw_dom->ctrl_val[i], m->res));
 }
 
-static void
-cat_wrmsr(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
+static void cat_wrmsr(struct msr_param *m)
 {
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(m->res);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(m->dom);
 	unsigned int i;
-	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 
 	for (i = m->low; i < m->high; i++)
 		wrmsrl(hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -384,7 +375,7 @@ void rdt_ctrl_update(void *arg)
 	struct msr_param *m = arg;
 
 	hw_res = resctrl_to_arch_res(m->res);
-	hw_res->msr_update(m->dom, m, m->res);
+	hw_res->msr_update(m);
 }
 
 /*
@@ -457,10 +448,11 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
 	hw_dom->ctrl_val = dc;
 	setup_default_ctrlval(r, dc);
 
+	m.res = r;
 	m.dom = d;
 	m.low = 0;
 	m.high = hw_res->num_closid;
-	hw_res->msr_update(d, &m, r);
+	hw_res->msr_update(&m);
 	return 0;
 }
 
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
index 165d8d4..b7291f6 100644
@@ -289,7 +289,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
 	msr_param.dom = d;
 	msr_param.low = idx;
 	msr_param.high = idx + 1;
-	hw_res->msr_update(d, &msr_param, r);
+	hw_res->msr_update(&msr_param);
 
 	return 0;
 }
arch/x86/kernel/cpu/resctrl/internal.h +1 -2
index ab2d315..f1d9268 100644
@@ -445,8 +445,7 @@ struct rdt_hw_resource {
 	struct rdt_resource	r_resctrl;
 	u32			num_closid;
 	unsigned int		msr_base;
-	void (*msr_update)	(struct rdt_domain *d, struct msr_param *m,
-				 struct rdt_resource *r);
+	void			(*msr_update)(struct msr_param *m);
 	unsigned int		mon_scale;
 	unsigned int		mbm_width;
 	unsigned int		mbm_cfg_mask;
maintainer
#96 lore raw

Re: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU

In reply to Tony Luck

Hi Tony,

Quoted 23 lines · Tony Luck

reset_all_ctrls() and resctrl_arch_update_domains() use on_each_cpu_mask() to call rdt_ctrl_update() on potentially one CPU from each domain.

But this means rdt_ctrl_update() needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely if cpus_lock is held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.

Commit c0d848fcb09d ("x86/resctrl: Remove lockdep annotation that triggers false positive") removed the incorrect assertions.

Add the target domain to the msr_param structure and
call rdt_ctrl_update() for each domain separately using smp_call_function_single(). This means that rdt_ctrl_update() doesn't need to search for the domain and get_domain_from_cpu() can safely assert that the cpus_lock is held since the remaining callers do not use IPI.

---
Changes since V4: Link: https://lore.kernel.org/all/[email protected]/

Quoted 7 lines

Reinette: Only assign "cpu" once in resctrl_arch_update_domains() [but see change from James below]

James: Use smp_call_function_any() instead of cpumask_any() + smp_call_function_single() to avoid unnecessary IPI in both resctrl_arch_update_domains() and reset_all_ctrls(). This eliminates a need for the "cpu" local variable.

Great catch, thank you for doing this.

Also, could you please stop referring to previous versions (in this case it even follows v3) using "In-Reply-To:"? Although v5 here follows v3, not v4. You previously [1] agreed and did so for v4 but not here.

Reinette

[1] https://lore.kernel.org/lkml/SJ1PR11MB60839AD3BC4DB15BF5B0BBE0FC562@SJ1PR11MB6083.namprd11.prod.outlook.com/

maintainer
#97 lore raw

RE: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU

Quoted 4 lines

Also, could you please stop referring to previous versions (in this case it even follows v3) using "In-Reply-To:"? Although v5 here follows v3, not v4. You previously [1] agreed and did so for v4 but not here.

Reinette

sorry. When sending a patch, or series I create a SEND shell script with the "git send-emal" command line with all the "--to" and "--cc" arguments to send to the right people and mailing lists.

I copied an old SEND script and forgot to delete the "--in-reply-to" argument from the script :-(

Will try to do better.

-Tony

maintainer
#98 lore raw

Re: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU

In reply to Tony Luck

Hi Boris,

Could you please consider this series for inclusion?

Thank you very much.

Reinette

Quoted 43 lines · Tony Luck

reset_all_ctrls() and resctrl_arch_update_domains() use on_each_cpu_mask() to call rdt_ctrl_update() on potentially one CPU from each domain.

But this means rdt_ctrl_update() needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely if cpus_lock is held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.

Commit c0d848fcb09d ("x86/resctrl: Remove lockdep annotation that triggers false positive") removed the incorrect assertions.

Add the target domain to the msr_param structure and
call rdt_ctrl_update() for each domain separately using smp_call_function_single(). This means that rdt_ctrl_update() doesn't need to search for the domain and get_domain_from_cpu() can safely assert that the cpus_lock is held since the remaining callers do not use IPI.

---
Changes since V4: Link: https://lore.kernel.org/all/[email protected]/

Reinette: Only assign "cpu" once in resctrl_arch_update_domains() [but see change from James below]

James: Use smp_call_function_any() instead of cpumask_any() + smp_call_function_single() to avoid unnecessary IPI in both resctrl_arch_update_domains() and reset_all_ctrls(). This eliminates a need for the "cpu" local variable.

Tony Luck (2):
x86/resctrl: Pass domain to target CPU
x86/resctrl: Simplify call convention for MSR update functions

 arch/x86/kernel/cpu/resctrl/internal.h    |  5 ++-
 arch/x86/kernel/cpu/resctrl/core.c        | 55 +++++++++--------------
 arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 40 ++++-------------
 arch/x86/kernel/cpu/resctrl/rdtgroup.c    | 12 +----
 4 files changed, 34 insertions(+), 78 deletions(-)

base-commit: c0d848fcb09d80a5f48b99f85e448185125ef59f

author reviewer
#99 lore raw
In reply to Reinette Chatre

Hi Tony, Reinette,

Quoted 7 lines · Reinette Chatre
Quoted 6 lines · Tony Luck
Quoted 5 lines · Thomas Gleixner
Quoted 3 lines · Borislav Petkov
Quoted 2 lines · James Morse

Hello!

Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)

The set applied ontop of tip:x86/cache gives:

Testing tip x86/cache that WARN fires while running tools/tests/selftests/resctrl/resctrl_test.

I evidently need to build a newer version of that tool.

Quoted 8 lines
Quoted 2 lines

Everthing runs OK if I drop the top commit:
fb700810d30b ("x86/resctrl: Separate arch and fs resctrl locks")

The new WARN_ON_ONCE() is why this encountered. The comment notes that lockdep_is_cpus_held() is used to determine if "someone is holding the CPUs lock" but it seems that lockdep_is_cpus_held() still only checks if "current" is holding cpu_hotplug_lock and that is not possible when running the code via IPI.

I was evidently mistaken that this was the difference between lockdep_is_cpus_held() and lockdep_assert_cpus_held().

It's a false positive, ripping out the check is the simplest thing to do.

Quoted 7 lines

So even though it is confirmed via lockdep_assert_cpus_held() that resctrl_arch_update_domains() holds cpu_hotplug_lock, it does not seem possible to have a similar lockdep check in the function called by it (resctrl_arch_update_domains()) via IPI. It thus does not look like that lockdep checking within get_domain_from_cpu() can be accurate and I cannot see what it can be replaced with to make it accurate. Any guidance will be appreciated. Perhaps we should just drop (but with detailed context comments remaining) the lockdep check in get_domain_from_cpu()?

Thanks,

James

maintainer
#100 lore raw

Hi James,

Quoted 14 lines · James Morse

Hi Tony, Reinette,

Quoted 11 lines · Reinette Chatre
Quoted 10 lines · Tony Luck
Quoted 9 lines · Thomas Gleixner
Quoted 7 lines · Borislav Petkov
Quoted 5 lines · James Morse

Hello!

It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)

Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line

The set applied ontop of tip:x86/cache gives:

Testing tip x86/cache that WARN fires while running tools/tests/selftests/resctrl/resctrl_test.

I evidently need to build a newer version of that tool.

There has been a lot of changes in the last few cycles. You can find the latest version on the "next" branch of the kselftest repo [1] where most recent enhancements are queued up for inclusion. If you are interested, there is one more series [2] pending merge to the kselftest repo, it adds a new test for the recent non-contiguous CBM support.

Reinette

[1] git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git
[2] https://lore.kernel.org/lkml/[email protected]/

maintainer
#101 lore raw
Quoted 10 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines

Testing tip x86/cache that WARN fires while running tools/tests/selftests/resctrl/resctrl_test.

I evidently need to build a newer version of that tool.

There has been a lot of changes in the last few cycles. You can find the latest version on the "next" branch of the kselftest repo [1] where most recent enhancements are queued up for inclusion. If you are interested, there is one more series [2] pending merge to the kselftest repo, it adds a new test for the recent non-contiguous CBM support.

James,

You didn't see this WARN because it isn't in your patch. Diff between what is in your tree and what was applied to TIP:

arch/x86/kernel/cpu/resctrl/core.c +2 -2
index 9f1aa555a8ea..8a4ef4f5bddc 100644
@@ -368,8 +368,8 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)
         * about locks this thread holds will lead to false positives. Check
         * someone is holding the CPUs lock.
         */
-       if (IS_ENABLED(CONFIG_LOCKDEP))
-               lockdep_is_cpus_held();
+       if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && IS_ENABLED(CONFIG_LOCKDEP))
+               WARN_ON_ONCE(!lockdep_is_cpus_held());
 
        list_for_each_entry(d, &r->domains, list) {
                /* Find the domain that contains this CPU */
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
index dc59643498bf..7997b47743a2 100644
@@ -567,7 +567,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
         * cpumask_any_housekeeping() prefers housekeeping CPUs, but
         * are all the CPUs nohz_full? If yes, pick a CPU to IPI.
         * MPAM's resctrl_arch_rmid_read() is unable to read the
-        * counters on some platforms if its called in irq context.
+        * counters on some platforms if its called in IRQ context.
         */
        if (tick_nohz_full_cpu(cpu))
                smp_call_function_any(&d->cpu_mask, mon_event_count, rr, 1);

-Tony

merge bot Applied
#102 lore raw

[tip: x86/cache] x86/resctrl: Remove lockdep annotation that triggers false positive

In reply to Tony Luck
The following commit has been merged into the x86/cache branch of tip:

The following commit has been merged into the x86/cache branch of tip:

Commit-ID:     c0d848fcb09d80a5f48b99f85e448185125ef59f
Gitweb:        https://git.kernel.org/tip/c0d848fcb09d80a5f48b99f85e448185125ef59f
Author:        James Morse <[email protected]>
AuthorDate:    Wed, 21 Feb 2024 12:23:06 
Committer:     Borislav Petkov (AMD) <[email protected]>
CommitterDate: Thu, 22 Feb 2024 16:15:38 +01:00

x86/resctrl: Remove lockdep annotation that triggers false positive

get_domain_from_cpu() walks a list of domains to find the one that contains the specified CPU. This needs to be protected against races with CPU hotplug when the list is modified. It has recently gained a lockdep annotation to check this.

The lockdep annotation causes false positives when called via IPI as the lock is held, but by another process. Remove it.

[ bp: Refresh it ontop of x86/cache. ]

Fixes fb700810d30b ("x86/resctrl: Separate arch and fs resctrl locks")
---
 arch/x86/kernel/cpu/resctrl/core.c |  9 ---------
 1 file changed, 9 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +0 -9
index 8a4ef4f..83e4034 100644
@@ -362,15 +362,6 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)
 {
 	struct rdt_domain *d;
 
-	/*
-	 * Walking r->domains, ensure it can't race with cpuhp.
-	 * Because this is called via IPI by rdt_ctrl_update(), assertions
-	 * about locks this thread holds will lead to false positives. Check
-	 * someone is holding the CPUs lock.
-	 */
-	if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && IS_ENABLED(CONFIG_LOCKDEP))
-		WARN_ON_ONCE(!lockdep_is_cpus_held());
-
 	list_for_each_entry(d, &r->domains, list) {
 		/* Find the domain that contains this CPU */
 		if (cpumask_test_cpu(cpu, &d->cpu_mask))
author reviewer
#103 lore raw
In reply to Borislav Petkov

Hi Boris,

Thanks for the config, (as Thomas pointed out) this is coming from an awkward lockdep annotation - turns out it also depends on HOTPLUG_CPU.

I'll post a v10 with the collected tags and this fixed.

Thanks,

James

Quoted 22 lines · Borislav Petkov
Quoted 5 lines · James Morse

Hello!

It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)

The set applied ontop of tip:x86/cache gives:

vmlinux.o: in function `get_domain_from_cpu':
(.text+0x150f33): undefined reference to `lockdep_is_cpus_held'
ld: vmlinux.o: in function `rdt_ctrl_update':
(.text+0x150fbc): undefined reference to `lockdep_is_cpus_held'

Config

00-14-04-randconfig-x86_64-26892.cfg

attached.

Thx.