x86/resctrl: monitored closid+rmid together, separate arch/fs locking
Subsystems (3)
[tip: x86/cache] x86/resctrl: Pass domain to target CPU
In reply to Tony LuckThe following commit has been merged into the x86/cache branch of tip:
The following commit has been merged into the x86/cache branch of tip:
Commit-ID: e3ca96e479c91d6ee657d3caa5092a6a3a620f9f Gitweb: https://git.kernel.org/tip/e3ca96e479c91d6ee657d3caa5092a6a3a620f9f Author: Tony Luck <[email protected]> AuthorDate: Fri, 08 Mar 2024 13:38:45 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 24 Apr 2024 13:41:41 +02:00
x86/resctrl: Pass domain to target CPU
reset_all_ctrls() and resctrl_arch_update_domains() use on_each_cpu_mask() to call rdt_ctrl_update() on potentially one CPU from each domain.
But this means rdt_ctrl_update() needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely if cpus_lock is held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.
Commit
c0d848fcb09d ("x86/resctrl: Remove lockdep annotation that triggers false positive")
removed the incorrect assertions.
Add the target domain to the msr_param structure and call rdt_ctrl_update() for each domain separately using smp_call_function_single(). This means that rdt_ctrl_update() doesn't need to search for the domain and get_domain_from_cpu() can safely assert that the cpus_lock is held since the remaining callers do not use IPI.
--- arch/x86/kernel/cpu/resctrl/core.c | 17 +++------- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 38 ++++------------------ arch/x86/kernel/cpu/resctrl/internal.h | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 12 +------ 4 files changed, 17 insertions(+), 52 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +6 -11
@@ -362,6 +362,8 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r) {struct rdt_domain *d; +lockdep_assert_cpus_held (); +list_for_each_entry (d, &r->domains, list) {/* Find the domain that contains this CPU */ if (cpumask_test_cpu (cpu, &d->cpu_mask))
@@ -378,19 +380,11 @@ u32 resctrl_arch_get_num_closid(struct rdt_resource *r)void rdt_ctrl_update (void *arg) { +struct rdt_hw_resource *hw_res;struct msr_param *m = arg; -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); -struct rdt_resource *r = m->res; -int cpu =smp_processor_id (); -struct rdt_domain *d; - d =get_domain_from_cpu (cpu, r); -if (d) { - hw_res->msr_update (d, m, r); -return ; - } -pr_warn_once ("cpu %d not found in any domain for resource %s\n" , - cpu, r->name); + hw_res =resctrl_to_arch_res (m->res); + hw_res->msr_update (m->dom, m, m->res); }/*
@@ -463,6 +457,7 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) hw_dom->ctrl_val = dc;setup_default_ctrlval (r, dc); + m.dom = d; m.low =0 ; m.high = hw_res->num_closid; hw_res->msr_update (d, &m, r);
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +7 -31
@@ -272,22 +272,6 @@ static u32 get_config_index(u32 closid, enum resctrl_conf_type type) } } -static bool apply_config (struct rdt_hw_domain *hw_dom, -struct resctrl_staged_config *cfg,u32 idx, -cpumask_var_t cpu_mask) -{ -struct rdt_domain *dom = &hw_dom->d_resctrl; - -if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) { -cpumask_set_cpu (cpumask_any (&dom->cpu_mask), cpu_mask); - hw_dom->ctrl_val[idx] = cfg->new_ctrl; - -return true; - } - -return false; -} -int resctrl_arch_update_one (struct rdt_resource *r,struct rdt_domain *d,u32 closid,enum resctrl_conf_type t,u32 cfg_val) {
@@ -302,6 +286,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, hw_dom->ctrl_val[idx] = cfg_val; msr_param.res = r; + msr_param.dom = d; msr_param.low = idx; msr_param.high = idx +1 ; hw_res->msr_update (d, &msr_param, r);
@@ -315,48 +300,39 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)struct rdt_hw_domain *hw_dom;struct msr_param msr_param;enum resctrl_conf_type t; -cpumask_var_t cpu_mask;struct rdt_domain *d;u32 idx;/* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held (); -if (!zalloc_cpumask_var (&cpu_mask,GFP_KERNEL )) -return -ENOMEM ; - - msr_param.res =NULL ;list_for_each_entry (d, &r->domains, list) { hw_dom =resctrl_to_arch_dom (d); + msr_param.res =NULL ;for (t =0 ; t <CDP_NUM_TYPES ; t++) { cfg = &hw_dom->d_resctrl.staged_config[t];if (!cfg->have_new_ctrl)continue ; idx =get_config_index (closid, t); -if (!apply_config (hw_dom, cfg, idx, cpu_mask)) +if (cfg->new_ctrl == hw_dom->ctrl_val[idx])continue ; + hw_dom->ctrl_val[idx] = cfg->new_ctrl;if (!msr_param.res) { msr_param.low = idx; msr_param.high = msr_param.low +1 ; msr_param.res = r; + msr_param.dom = d; }else { msr_param.low =min (msr_param.low, idx); msr_param.high =max (msr_param.high, idx +1 ); } } +if (msr_param.res) +smp_call_function_any (&d->cpu_mask, rdt_ctrl_update, &msr_param,1 ); } -if (cpumask_empty (cpu_mask)) - goto done; - -/* Update resource control msr on all the CPUs. */ -on_each_cpu_mask (cpu_mask, rdt_ctrl_update, &msr_param,1 ); - -done: -free_cpumask_var (cpu_mask); -return 0 ; }
arch/x86/kernel/cpu/resctrl/internal.h +2 -0
@@ -379,11 +379,13 @@ static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r)/** * struct msr_param - set a range of MSRs from a domain * @res: The resource to use +* @dom: The domain to update * @low: Beginning index from base MSR * @high: End index */ struct msr_param {struct rdt_resource *res; +struct rdt_domain *dom;u32 low;u32 high; };
arch/x86/kernel/cpu/resctrl/rdtgroup.c +2 -10
@@ -2813,16 +2813,12 @@ static int reset_all_ctrls(struct rdt_resource *r)struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);struct rdt_hw_domain *hw_dom;struct msr_param msr_param; -cpumask_var_t cpu_mask;struct rdt_domain *d;int i;/* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held (); -if (!zalloc_cpumask_var (&cpu_mask,GFP_KERNEL )) -return -ENOMEM ; - msr_param.res = r; msr_param.low =0 ; msr_param.high = hw_res->num_closid;
@@ -2834,17 +2830,13 @@ static int reset_all_ctrls(struct rdt_resource *r)*/ list_for_each_entry (d, &r->domains, list) { hw_dom =resctrl_to_arch_dom (d); -cpumask_set_cpu (cpumask_any (&d->cpu_mask), cpu_mask);for (i =0 ; i < hw_res->num_closid; i++) hw_dom->ctrl_val[i] = r->default_ctrl; + msr_param.dom = d; +smp_call_function_any (&d->cpu_mask, rdt_ctrl_update, &msr_param,1 ); } -/* Update CBM on all the CPUs in cpu_mask */ -on_each_cpu_mask (cpu_mask, rdt_ctrl_update, &msr_param,1 ); - -free_cpumask_var (cpu_mask); -return 0 ; }
Tony Luck
maintainerPATCH v5 2/2 x86/resctrl: Simplify call convention for MSR update functions
In reply to Tony LuckThe per-resource MSR update functions cat_wrmsr(), mba_wrmsr_intel(), and mba_wrmsr_amd() all take three arguments:
(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
struct msr_param contains pointers to both struct rdt_resource and struct rdt_domain, thus only struct msr_param is necessary.
Pass struct msr_param as a single parameter. Clean up formatting and fix some fir tree declaration ordering.
No functional change.
--- arch/x86/kernel/cpu/resctrl/internal.h | 3 +- arch/x86/kernel/cpu/resctrl/core.c | 40 +++++++++-------------- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 2 +- 3 files changed, 18 insertions(+), 27 deletions(-)
arch/x86/kernel/cpu/resctrl/internal.h +1 -2
@@ -444,8 +444,7 @@ struct rdt_hw_resource {struct rdt_resource r_resctrl;u32 num_closid;unsigned int msr_base; -void (*msr_update) (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); +void (*msr_update)(struct msr_param *m);unsigned int mon_scale;unsigned int mbm_width;unsigned int mbm_cfg_mask;
arch/x86/kernel/cpu/resctrl/core.c +16 -24
@@ -56,14 +56,9 @@ int max_name_width, max_data_width;*/ bool rdt_alloc_capable; -static void -mba_wrmsr_intel (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); -static void -cat_wrmsr (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r); -static void -mba_wrmsr_amd (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); +static void mba_wrmsr_intel (struct msr_param *m); +static void cat_wrmsr (struct msr_param *m); +static void mba_wrmsr_amd (struct msr_param *m);#define domain_init (id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)
@@ -309,12 +304,11 @@ static void rdt_get_cdp_l2_config(void)rdt_get_cdp_config (RDT_RESOURCE_L2 ); } -static void -mba_wrmsr_amd (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r) +static void mba_wrmsr_amd (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);for (i = m->low; i < m->high; i++)wrmsrl (hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -334,25 +328,22 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)return r->default_ctrl; } -static void -mba_wrmsr_intel (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r) +static void mba_wrmsr_intel (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);/* Write the delay values for mba. */ for (i = m->low; i < m->high; i++) -wrmsrl (hw_res->msr_base + i,delay_bw_map (hw_dom->ctrl_val[i], r)); +wrmsrl (hw_res->msr_base + i,delay_bw_map (hw_dom->ctrl_val[i], m->res)); } -static void -cat_wrmsr (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r) +static void cat_wrmsr (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);for (i = m->low; i < m->high; i++)wrmsrl (hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -384,7 +375,7 @@ void rdt_ctrl_update(void *arg)struct msr_param *m = arg; hw_res =resctrl_to_arch_res (m->res); - hw_res->msr_update (m->dom, m, m->res); + hw_res->msr_update (m); }/*
@@ -457,10 +448,11 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) hw_dom->ctrl_val = dc;setup_default_ctrlval (r, dc); + m.res = r; m.dom = d; m.low =0 ; m.high = hw_res->num_closid; - hw_res->msr_update (d, &m, r); + hw_res->msr_update (&m);return 0 ; }
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
@@ -289,7 +289,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, msr_param.dom = d; msr_param.low = idx; msr_param.high = idx +1 ; - hw_res->msr_update (d, &msr_param, r); + hw_res->msr_update (&msr_param);return 0 ; }
Signature · 1 line
2.43.0
Re: [PATCH v5 2/2] x86/resctrl: Simplify call convention for MSR update functions
Quoted 11 lines · Tony Luck
The per-resource MSR update functions
cat_wrmsr(),mba_wrmsr_intel(), andmba_wrmsr_amd()all take three arguments:(struct
rdt_domain*d, structmsr_param*m, structrdt_resource*r)struct
msr_paramcontains pointers to both structrdt_resourceand structrdt_domain, thus only structmsr_paramis necessary.Pass struct
msr_paramas a single parameter. Clean up formatting and fix some fir tree declaration ordering.
Quoted 6 lines
No functional change.
Suggested-by Reinette Chatre <[email protected]>Signed-off-by Tony Luck <[email protected]>Reviewed-by Reinette Chatre <[email protected]>
[tip: x86/cache] x86/resctrl: Simplify call convention for MSR update functions
In reply to Tony LuckThe following commit has been merged into the x86/cache branch of tip:
The following commit has been merged into the x86/cache branch of tip:
Commit-ID: bd4955d4bc2182ccb660c9c30a4dd7f36feaf943 Gitweb: https://git.kernel.org/tip/bd4955d4bc2182ccb660c9c30a4dd7f36feaf943 Author: Tony Luck <[email protected]> AuthorDate: Fri, 08 Mar 2024 13:38:46 -08:00 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Wed, 24 Apr 2024 13:47:00 +02:00
x86/resctrl: Simplify call convention for MSR update functions
The per-resource MSR update functions cat_wrmsr(), mba_wrmsr_intel(), and mba_wrmsr_amd() all take three arguments:
(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
struct msr_param contains pointers to both struct rdt_resource and struct rdt_domain, thus only struct msr_param is necessary.
Pass struct msr_param as a single parameter. Clean up formatting and fix some fir tree declaration ordering.
No functional change.
--- arch/x86/kernel/cpu/resctrl/core.c | 40 ++++++++-------------- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 2 +- arch/x86/kernel/cpu/resctrl/internal.h | 3 +-- 3 files changed, 18 insertions(+), 27 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +16 -24
@@ -56,14 +56,9 @@ int max_name_width, max_data_width;*/ bool rdt_alloc_capable; -static void -mba_wrmsr_intel (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); -static void -cat_wrmsr (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r); -static void -mba_wrmsr_amd (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); +static void mba_wrmsr_intel (struct msr_param *m); +static void cat_wrmsr (struct msr_param *m); +static void mba_wrmsr_amd (struct msr_param *m);#define domain_init (id) LIST_HEAD_INIT(rdt_resources_all[id].r_resctrl.domains)
@@ -309,12 +304,11 @@ static void rdt_get_cdp_l2_config(void)rdt_get_cdp_config (RDT_RESOURCE_L2 ); } -static void -mba_wrmsr_amd (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r) +static void mba_wrmsr_amd (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);for (i = m->low; i < m->high; i++)wrmsrl (hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -334,25 +328,22 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)return r->default_ctrl; } -static void -mba_wrmsr_intel (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r) +static void mba_wrmsr_intel (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);/* Write the delay values for mba. */ for (i = m->low; i < m->high; i++) -wrmsrl (hw_res->msr_base + i,delay_bw_map (hw_dom->ctrl_val[i], r)); +wrmsrl (hw_res->msr_base + i,delay_bw_map (hw_dom->ctrl_val[i], m->res)); } -static void -cat_wrmsr (struct rdt_domain *d,struct msr_param *m,struct rdt_resource *r) +static void cat_wrmsr (struct msr_param *m) { +struct rdt_hw_resource *hw_res =resctrl_to_arch_res (m->res); +struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (m->dom);unsigned int i; -struct rdt_hw_domain *hw_dom =resctrl_to_arch_dom (d); -struct rdt_hw_resource *hw_res =resctrl_to_arch_res (r);for (i = m->low; i < m->high; i++)wrmsrl (hw_res->msr_base + i, hw_dom->ctrl_val[i]);
@@ -384,7 +375,7 @@ void rdt_ctrl_update(void *arg)struct msr_param *m = arg; hw_res =resctrl_to_arch_res (m->res); - hw_res->msr_update (m->dom, m, m->res); + hw_res->msr_update (m); }/*
@@ -457,10 +448,11 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) hw_dom->ctrl_val = dc;setup_default_ctrlval (r, dc); + m.res = r; m.dom = d; m.low =0 ; m.high = hw_res->num_closid; - hw_res->msr_update (d, &m, r); + hw_res->msr_update (&m);return 0 ; }
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
@@ -289,7 +289,7 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, msr_param.dom = d; msr_param.low = idx; msr_param.high = idx +1 ; - hw_res->msr_update (d, &msr_param, r); + hw_res->msr_update (&msr_param);return 0 ; }
arch/x86/kernel/cpu/resctrl/internal.h +1 -2
@@ -445,8 +445,7 @@ struct rdt_hw_resource {struct rdt_resource r_resctrl;u32 num_closid;unsigned int msr_base; -void (*msr_update) (struct rdt_domain *d,struct msr_param *m, -struct rdt_resource *r); +void (*msr_update)(struct msr_param *m);unsigned int mon_scale;unsigned int mbm_width;unsigned int mbm_cfg_mask;
Reinette Chatre
maintainerRe: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU
In reply to Tony LuckHi Tony,
Quoted 23 lines · Tony Luck
reset_all_ctrls()andresctrl_arch_update_domains()useon_each_cpu_mask()to callrdt_ctrl_update()on potentially one CPU from each domain.But this means
rdt_ctrl_update()needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely ifcpus_lockis held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.Commit
c0d848fcb09d("x86/resctrl: Remove lockdep annotation that triggers false positive") removed the incorrect assertions.Add the target domain to the
msr_paramstructure and
callrdt_ctrl_update()for each domain separately usingsmp_call_function_single(). This means thatrdt_ctrl_update()doesn't need to search for the domain andget_domain_from_cpu()can safely assert that thecpus_lockis held since the remaining callers do not use IPI.Signed-off-by Tony Luck <[email protected]>---
Changes since V4: Link: https://lore.kernel.org/all/[email protected]/
Quoted 7 lines
Reinette: Only assign "cpu" once in
resctrl_arch_update_domains()[but see change from James below]James: Use
smp_call_function_any()instead ofcpumask_any()+smp_call_function_single()to avoid unnecessary IPI in bothresctrl_arch_update_domains()andreset_all_ctrls(). This eliminates a need for the "cpu" local variable.
Great catch, thank you for doing this.
Also, could you please stop referring to previous versions (in this case it even follows v3) using "In-Reply-To:"? Although v5 here follows v3, not v4. You previously [1] agreed and did so for v4 but not here.
Reinette
Luck, Tony
maintainerRE: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU
Quoted 4 lines
Also, could you please stop referring to previous versions (in this case it even follows v3) using "In-Reply-To:"? Although v5 here follows v3, not v4. You previously [1] agreed and did so for v4 but not here.
Reinette
sorry. When sending a patch, or series I create a SEND shell script with the "git send-emal" command line with all the "--to" and "--cc" arguments to send to the right people and mailing lists.
I copied an old SEND script and forgot to delete the "--in-reply-to" argument from the script :-(
Will try to do better.
-Tony
Reinette Chatre
maintainerRe: [PATCH v5 0/2] x86/resctrl: Pass domain to target CPU
In reply to Tony LuckHi Boris,
Could you please consider this series for inclusion?
Thank you very much.
Reinette
Quoted 43 lines · Tony Luck
reset_all_ctrls()andresctrl_arch_update_domains()useon_each_cpu_mask()to callrdt_ctrl_update()on potentially one CPU from each domain.But this means
rdt_ctrl_update()needs to figure out which domain to apply changes to. Doing so requires a search of all domains in a resource, which can only be done safely ifcpus_lockis held. Both callers do hold this lock, but there isn't a way for a function called on another CPU via IPI to verify this.Commit
c0d848fcb09d("x86/resctrl: Remove lockdep annotation that triggers false positive") removed the incorrect assertions.Add the target domain to the
msr_paramstructure and
callrdt_ctrl_update()for each domain separately usingsmp_call_function_single(). This means thatrdt_ctrl_update()doesn't need to search for the domain andget_domain_from_cpu()can safely assert that thecpus_lockis held since the remaining callers do not use IPI.Signed-off-by Tony Luck <[email protected]>---
Changes since V4: Link: https://lore.kernel.org/all/[email protected]/Reinette: Only assign "cpu" once in
resctrl_arch_update_domains()[but see change from James below]James: Use
smp_call_function_any()instead ofcpumask_any()+smp_call_function_single()to avoid unnecessary IPI in bothresctrl_arch_update_domains()andreset_all_ctrls(). This eliminates a need for the "cpu" local variable.Tony Luck (2):
x86/resctrl: Pass domain to target CPU
x86/resctrl: Simplify call convention for MSR update functionsarch/x86/kernel/cpu/resctrl/internal.h | 5 ++- arch/x86/kernel/cpu/resctrl/core.c | 55 +++++++++-------------- arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 40 ++++------------- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 12 +---- 4 files changed, 34 insertions(+), 78 deletions(-)base-commit:
c0d848fcb09d80a5f48b99f85e448185125ef59f
James Morse
reviewerHi Tony, Reinette,
Quoted 7 lines · Reinette Chatre
Quoted 6 lines · Tony Luck
Quoted 5 lines · Thomas Gleixner
Quoted 3 lines · Borislav Petkov
Quoted 2 lines · James Morse
Hello!
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)The set applied ontop of tip:x86/cache gives:
Testing tip x86/cache that WARN fires while running
tools/tests/selftests/resctrl/resctrl_test.
I evidently need to build a newer version of that tool.
Quoted 8 lines
Quoted 2 lines
Everthing runs OK if I drop the top commit:
fb700810d30b("x86/resctrl: Separate arch and fs resctrl locks")The new
WARN_ON_ONCE()is why this encountered. The comment notes thatlockdep_is_cpus_held()is used to determine if "someone is holding the CPUs lock" but it seems thatlockdep_is_cpus_held()still only checks if "current" is holdingcpu_hotplug_lockand that is not possible when running the code via IPI.
I was evidently mistaken that this was the difference between lockdep_is_cpus_held() and lockdep_assert_cpus_held().
It's a false positive, ripping out the check is the simplest thing to do.
Quoted 7 lines
So even though it is confirmed via
lockdep_assert_cpus_held()thatresctrl_arch_update_domains()holdscpu_hotplug_lock, it does not seem possible to have a similar lockdep check in the function called by it (resctrl_arch_update_domains()) via IPI. It thus does not look like that lockdep checking withinget_domain_from_cpu()can be accurate and I cannot see what it can be replaced with to make it accurate. Any guidance will be appreciated. Perhaps we should just drop (but with detailed context comments remaining) the lockdep check inget_domain_from_cpu()?
Thanks,
James
Reinette Chatre
maintainerHi James,
Quoted 14 lines · James Morse
Hi Tony, Reinette,
Quoted 11 lines · Reinette Chatre
Quoted 10 lines · Tony Luck
Quoted 9 lines · Thomas Gleixner
Quoted 7 lines · Borislav Petkov
Quoted 5 lines · James Morse
Hello!
It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
The set applied ontop of tip:x86/cache gives:
Testing tip x86/cache that WARN fires while running
tools/tests/selftests/resctrl/resctrl_test.I evidently need to build a newer version of that tool.
There has been a lot of changes in the last few cycles. You can find the latest version on the "next" branch of the kselftest repo [1] where most recent enhancements are queued up for inclusion. If you are interested, there is one more series [2] pending merge to the kselftest repo, it adds a new test for the recent non-contiguous CBM support.
Reinette
[1] git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git
[2] https://lore.kernel.org/lkml/[email protected]/
Luck, Tony
maintainerQuoted 10 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Testing tip x86/cache that WARN fires while running
tools/tests/selftests/resctrl/resctrl_test.I evidently need to build a newer version of that tool.
There has been a lot of changes in the last few cycles. You can find the latest version on the "next" branch of the kselftest repo [1] where most recent enhancements are queued up for inclusion. If you are interested, there is one more series [2] pending merge to the kselftest repo, it adds a new test for the recent non-contiguous CBM support.
James,
You didn't see this WARN because it isn't in your patch. Diff between what is in your tree and what was applied to TIP:
arch/x86/kernel/cpu/resctrl/core.c +2 -2
@@ -368,8 +368,8 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r)* about locks this thread holds will lead to false positives. Check * someone is holding the CPUs lock. */ -if (IS_ENABLED (CONFIG_LOCKDEP )) -lockdep_is_cpus_held (); +if (IS_ENABLED (CONFIG_HOTPLUG_CPU ) &&IS_ENABLED (CONFIG_LOCKDEP )) +WARN_ON_ONCE (!lockdep_is_cpus_held ());list_for_each_entry (d, &r->domains, list) {/* Find the domain that contains this CPU */
arch/x86/kernel/cpu/resctrl/ctrlmondata.c +1 -1
@@ -567,7 +567,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,* cpumask_any_housekeeping() prefers housekeeping CPUs, but * are all the CPUs nohz_full? If yes, pick a CPU to IPI. * MPAM's resctrl_arch_rmid_read() is unable to read the -* counters on some platforms if its called in irq context. +* counters on some platforms if its called in IRQ context. */ if (tick_nohz_full_cpu (cpu))smp_call_function_any (&d->cpu_mask, mon_event_count, rr,1 );
-Tony
[tip: x86/cache] x86/resctrl: Remove lockdep annotation that triggers false positive
In reply to Tony LuckThe following commit has been merged into the x86/cache branch of tip:
The following commit has been merged into the x86/cache branch of tip:
Commit-ID: c0d848fcb09d80a5f48b99f85e448185125ef59f Gitweb: https://git.kernel.org/tip/c0d848fcb09d80a5f48b99f85e448185125ef59f Author: James Morse <[email protected]> AuthorDate: Wed, 21 Feb 2024 12:23:06 Committer: Borislav Petkov (AMD) <[email protected]> CommitterDate: Thu, 22 Feb 2024 16:15:38 +01:00
x86/resctrl: Remove lockdep annotation that triggers false positive
get_domain_from_cpu() walks a list of domains to find the one that contains the specified CPU. This needs to be protected against races with CPU hotplug when the list is modified. It has recently gained a lockdep annotation to check this.
The lockdep annotation causes false positives when called via IPI as the lock is held, but by another process. Remove it.
[ bp: Refresh it ontop of x86/cache. ]
--- arch/x86/kernel/cpu/resctrl/core.c | 9 --------- 1 file changed, 9 deletions(-)
arch/x86/kernel/cpu/resctrl/core.c +0 -9
@@ -362,15 +362,6 @@ struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r) {struct rdt_domain *d; -/* -* Walking r->domains, ensure it can't race with cpuhp. -* Because this is called via IPI by rdt_ctrl_update(), assertions -* about locks this thread holds will lead to false positives. Check -* someone is holding the CPUs lock. -*/ -if (IS_ENABLED (CONFIG_HOTPLUG_CPU ) &&IS_ENABLED (CONFIG_LOCKDEP )) -WARN_ON_ONCE (!lockdep_is_cpus_held ()); -list_for_each_entry (d, &r->domains, list) {/* Find the domain that contains this CPU */ if (cpumask_test_cpu (cpu, &d->cpu_mask))
James Morse
reviewerHi Boris,
Thanks for the config, (as Thomas pointed out) this is coming from an awkward lockdep annotation - turns out it also depends on HOTPLUG_CPU.
I'll post a v10 with the collected tags and this fixed.
Thanks,
James
Quoted 22 lines · Borislav Petkov
Quoted 5 lines · James Morse
Hello!
It's been back and forth for whether this series should be rebased onto Tony's SNC series. This version isn't, its based on tip/x86/cache.
(I have the rebased-and-tested versions if anyone needs them)The set applied ontop of tip:x86/cache gives:
vmlinux.o: in function `get_domain_from_cpu':
(.text+0x150f33): undefined reference to `lockdep_is_cpus_held'
ld: vmlinux.o: in function `rdt_ctrl_update':
(.text+0x150fbc): undefined reference to `lockdep_is_cpus_held'Config
00-14-04-randconfig-x86_64-26892.cfg
attached.
Thx.