Skip to contentSkip to conversation

x86/acpi: fix a deadlock with cpu hotplug

PATCH Superseded Qian Cai 2 participants 2 messages Mar 29, 2020

Subsystems (3)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Qian Cai

author
#0 lore raw

PATCH 1/1 x86/acpi: fix a deadlock with cpu hotplug

Similar to the commit 0266d81e9bf5 ("acpi/processor: Prevent cpu hotplug deadlock") except this is for acpi_processor_ffh_cstate_probe():

"The problem is that the work is scheduled on the current CPU from the hotplug thread associated with that CPU.

It's not required to invoke these functions via the workqueue because the hotplug thread runs on the target CPU already.

Check whether current is a per cpu thread pinned on the target CPU and invoke the function directly to avoid the workqueue."

Since CONFIG_ACPI_PROCESSOR (for cstate.c) selects CONFIG_ACPI_CPU_FREQ_PSS (for processor_throttling.c) on x86, just declare call_on_cpu() from for processor_throttling.c as an external function and use it in cstate.c.

WARNING: possible circular locking dependency detected
------------------------------------------------------
cpuhp/1/15 is trying to acquire lock:
ffffc90003447a28 ((work_completion)(&wfc.work)){+.+.}-{0:0}, at: __flush_work+0x4c6/0x630

but task is already holding lock:
ffffffffafa1c0e8 (cpuidle_lock){+.+.}-{3:3}, at: cpuidle_pause_and_lock+0x17/0x20

which lock already depends on the new lock.

the existing dependency chain (in reverse order) is:

-> #1 (cpu_hotplug_lock){++++}-{0:0}:

 cpus_read_lock+0x3e/0xc0
 irq_calc_affinity_vectors+0x5f/0x91
 __pci_enable_msix_range+0x10f/0x9a0
 pci_alloc_irq_vectors_affinity+0x13e/0x1f0
 pci_alloc_irq_vectors_affinity at drivers/pci/msi.c:1208
 pqi_ctrl_init+0x72f/0x1618 [smartpqi]
 pqi_pci_probe.cold.63+0x882/0x892 [smartpqi]
 local_pci_probe+0x7a/0xc0
 work_for_cpu_fn+0x2e/0x50
 process_one_work+0x57e/0xb90
 worker_thread+0x363/0x5b0
 kthread+0x1f4/0x220
 ret_from_fork+0x27/0x50

-> #0 ((work_completion)(&wfc.work)){+.+.}-{0:0}:

 __lock_acquire+0x2244/0x32a0
 lock_acquire+0x1a2/0x680
 __flush_work+0x4e6/0x630
 work_on_cpu+0x114/0x160
 acpi_processor_ffh_cstate_probe+0x129/0x250
 acpi_processor_evaluate_cst+0x4c8/0x580
 acpi_processor_get_power_info+0x86/0x740
 acpi_processor_hotplug+0xc3/0x140
 acpi_soft_cpu_online+0x102/0x1d0
 cpuhp_invoke_callback+0x197/0x1120
 cpuhp_thread_fun+0x252/0x2f0
 smpboot_thread_fn+0x255/0x440
 kthread+0x1f4/0x220
 ret_from_fork+0x27/0x50

other info that might help us debug this:

 Chain exists of:
 (work_completion)(&wfc.work) --> cpuhp_state-up --> cpuidle_lock

 Possible unsafe locking scenario:

CPU0 CPU1
---- ----

 lock(cpuidle_lock);
                         lock(cpuhp_state-up);
                         lock(cpuidle_lock);

lock((work_completion)(&wfc.work));

*** DEADLOCK ***

3 locks held by cpuhp/1/15:
#0: ffffffffaf51ab10 (cpu_hotplug_lock){++++}-{0:0}, at: cpuhp_thread_fun+0x69/0x2f0 #1: ffffffffaf51ad40 (cpuhp_state-up){+.+.}-{0:0}, at: cpuhp_thread_fun+0x69/0x2f0 #2: ffffffffafa1c0e8 (cpuidle_lock){+.+.}-{3:3}, at: cpuidle_pause_and_lock+0x17/0x20

 Call Trace:
 dump_stack+0xa0/0xea
 print_circular_bug.cold.52+0x147/0x14c
 check_noncircular+0x295/0x2d0
 __lock_acquire+0x2244/0x32a0
 lock_acquire+0x1a2/0x680
 __flush_work+0x4e6/0x630
 work_on_cpu+0x114/0x160
 acpi_processor_ffh_cstate_probe+0x129/0x250
 acpi_processor_evaluate_cst+0x4c8/0x580
 acpi_processor_get_power_info+0x86/0x740
 acpi_processor_hotplug+0xc3/0x140
 acpi_soft_cpu_online+0x102/0x1d0
 cpuhp_invoke_callback+0x197/0x1120
 cpuhp_thread_fun+0x252/0x2f0
 smpboot_thread_fn+0x255/0x440
 kthread+0x1f4/0x220
 ret_from_fork+0x27/0x50
---
 arch/x86/kernel/acpi/cstate.c       | 3 ++-
 drivers/acpi/processor_throttling.c | 2 +-
 include/acpi/processor.h            | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)
arch/x86/kernel/acpi/cstate.c +2 -1
index caf2edccbad2..49ae4e1ac9cd 100644
@@ -161,7 +161,8 @@ int acpi_processor_ffh_cstate_probe(unsigned int cpu,
 
 	/* Make sure we are running on right CPU */
 
-	retval = work_on_cpu(cpu, acpi_processor_ffh_cstate_probe_cpu, cx);
+	retval = call_on_cpu(cpu, acpi_processor_ffh_cstate_probe_cpu, cx,
+			     false);
 	if (retval == 0) {
 		/* Use the hint in CST */
 		percpu_entry->states[cx->index].eax = cx->address;
drivers/acpi/processor_throttling.c +1 -1
index 532a1ae3595a..122f63cacd86 100644
@@ -897,7 +897,7 @@ static long __acpi_processor_get_throttling(void *data)
 	return pr->throttling.acpi_processor_get_throttling(pr);
 }
 
-static int call_on_cpu(int cpu, long (*fn)(void *), void *arg, bool direct)
+int call_on_cpu(int cpu, long (*fn)(void *), void *arg, bool direct)
 {
 	if (direct || (is_percpu_thread() && cpu == smp_processor_id()))
 		return fn(arg);
include/acpi/processor.h +3 -0
index 47805172e73d..9569b3467553 100644
@@ -297,6 +297,9 @@ static inline void acpi_processor_ffh_cstate_enter(struct acpi_processor_cx
 }
 #endif
 
+/* in processor_throttling.c */
+int call_on_cpu(int cpu, long (*fn)(void *), void *arg, bool direct);
+
 /* in processor_perflib.c */
 
 #ifdef CONFIG_CPU_FREQ
Signature · 1 line
2.21.0 (Apple Git-122.2)
#1 lore raw
Thank you for the patch! Yet something to improve:

Hi Qian,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on pm/linux-next]
[also build test ERROR on tip/auto-latest tip/x86/core v5.6-rc7 next-20200327] [if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Qian-Cai/x86-acpi-fix-a-deadlock-with-cpu-hotplug/20200329-090840
base:   https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git linux-next
config: x86_64-randconfig-b001-20200329 (attached as .config)
compiler: gcc-7 (Debian 7.5.0-5) 7.5.0
reproduce:
        # save the attached .config to linux build tree
        make ARCH=x86_64 

If you fix the issue, kindly add following tag

All errors (new ones prefixed by >>):

ld: arch/x86/kernel/acpi/cstate.o: in function `acpi_processor_ffh_cstate_probe':

>> arch/x86/kernel/acpi/cstate.c:164: undefined reference to `call_on_cpu'

vim +164 arch/x86/kernel/acpi/cstate.c

144
145 int acpi_processor_ffh_cstate_probe(unsigned int cpu,
146 struct acpi_processor_cx *cx, struct acpi_power_register *reg)
147 {
148 struct cstate_entry *percpu_entry;
149 struct cpuinfo_x86 *c = &cpu_data(cpu);
150 long retval;
151
152 if (!cpu_cstate_entry || c->cpuid_level < CPUID_MWAIT_LEAF)
153 return -1;
154
155 if (reg->bit_offset != NATIVE_CSTATE_BEYOND_HALT)
156 return -1;
157
158 percpu_entry = per_cpu_ptr(cpu_cstate_entry, cpu);
159 percpu_entry->states[cx->index].eax = 0;
160 percpu_entry->states[cx->index].ecx = 0;
161
162 /* Make sure we are running on right CPU */
163

 > 164		retval = call_on_cpu(cpu, acpi_processor_ffh_cstate_probe_cpu, cx,

165 false);
166 if (retval == 0) {
167 /* Use the hint in CST */
168 percpu_entry->states[cx->index].eax = cx->address;
169 percpu_entry->states[cx->index].ecx = MWAIT_ECX_INTERRUPT_BREAK;
170 }
171
172 /*
173 * For _CST FFH on Intel, if GAS.access_size bit 1 is cleared,
174 * then we should skip checking BM_STS for this C-state.
175 * ref: "Intel Processor Vendor-Specific ACPI Interface Specification"
176 */
177 if ((c->x86_vendor == X86_VENDOR_INTEL) && !(reg->access_size & 0x2))
178 cx->bm_sts_skip = 1;
179
180 return retval;
181 }
182 EXPORT_SYMBOL_GPL(acpi_processor_ffh_cstate_probe);
183

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org