Skip to contentSkip to conversation

bug in cpuid & msr on nosmp machine

PATCH In review matthieu castet 5 participants 9 messages May 19, 2004 – May 21, 2004

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 bug in cpuid & msr on nosmp machine

hi,

on monocpu machine (and maybe even on smp machine), when you try to acces to a cpu that don't exist (/dev/cpu/1/cpuid), cpuid (msr) call cpu_online, but on nosmp machine if the cpu!=0 this procude a BUG(); So I add a check that verify if the cpu can exist before calling cpu_online.

Matthieu CASTET

arch/i386/kernel/cpuid.c +1 -1
@@ -135,7 +135,7 @@
   int cpu = iminor(file->f_dentry->d_inode);
   struct cpuinfo_x86 *c = &(cpu_data)[cpu];
 
-  if (cpu >= num_possible_cpus() || !cpu_online(cpu))
+  if (!cpu_online(cpu))
     return -ENXIO;		/* No such CPU */
   if ( c->cpuid_level < 0 )
     return -EIO;		/* CPUID not supported */
arch/i386/kernel/msr.c +1 -1
@@ -241,7 +241,7 @@
   int cpu = iminor(file->f_dentry->d_inode);
   struct cpuinfo_x86 *c = &(cpu_data)[cpu];
   
-  if (cpu >= num_possible_cpus() || !cpu_online(cpu))
+  if (!cpu_online(cpu))
     return -ENXIO;		/* No such CPU */
   if ( !cpu_has(c, X86_FEATURE_MSR) )
     return -EIO;		/* MSR not supported */
#1 lore raw
Quoted 6 lines · matthieu castet

hi,

on monocpu machine (and maybe even on smp machine), when you try to acces to a cpu that don't exist (/dev/cpu/1/cpuid), cpuid (msr) call cpu_online, but on nosmp machine if the cpu!=0 this procude a BUG(); So I add a check that verify if the cpu can exist before calling cpu_online.

Matthieu CASTET

oups i send the wrong patch

arch/i386/kernel/cpuid.c +1 -1
@@ -135,7 +135,7 @@
   int cpu = iminor(file->f_dentry->d_inode);
   struct cpuinfo_x86 *c = &(cpu_data)[cpu];
 
-  if (!cpu_online(cpu))
+  if (cpu >= num_possible_cpus() || !cpu_online(cpu))
     return -ENXIO;		/* No such CPU */
   if ( c->cpuid_level < 0 )
     return -EIO;		/* CPUID not supported */
arch/i386/kernel/msr.c +1 -1
@@ -241,7 +241,7 @@
   int cpu = iminor(file->f_dentry->d_inode);
   struct cpuinfo_x86 *c = &(cpu_data)[cpu];
   
-  if (!cpu_online(cpu))
+  if (cpu >= num_possible_cpus() || !cpu_online(cpu))
     return -ENXIO;		/* No such CPU */
   if ( !cpu_has(c, X86_FEATURE_MSR) )
     return -EIO;		/* MSR not supported */
#2 lore raw
In reply to matthieu castet

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Quoted 7 lines · matthieu castet

hi,

on monocpu machine (and maybe even on smp machine), when you try to acces to a cpu that don't exist (/dev/cpu/1/cpuid), cpuid (msr) call cpu_online, but on nosmp machine if the cpu!=0 this procude a BUG(); So I add a check that verify if the cpu can exist before calling cpu_online.

Added a check? I think your patch is reversed.

Jeff.

- -  if (cpu >= num_possible_cpus() || !cpu_online(cpu))
+  if (!cpu_online(cpu))
     return -ENXIO;             /* No such CPU */

- --
Hegh QaQ law'
quvHa'ghach QaQ puS
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.2.4 (GNU/Linux)

iD8DBQFArBuwwFP0+seVj/4RArcUAJwNIuZ6RhDYaHfnLIjKkiyXrXjfgwCgytvY hpqUVuBbNwragYDMkjNOm4w=
=8DZk
-----END PGP SIGNATURE-----

#5 lore raw
In reply to Andrew Morton
Quoted 4 lines · Andrew Morton

I think what you want here is

	if (!cpu_possible(cpu) || !cpu_online(cpu))
		return -ENXIO;

It works, but it's not really correct. cpu_possible() is correct, but cpu_online() might no longer be true by the time do_cpu_read() calls do_cpu_id().

One way would be to do lock_cpu_hotplug() in cpuid_open() and introduce a cpuid_close() which would do unlock_cpu_hotplug(). Another would be to drop the check here, and fail the actual read. A final way is to do no checks, in which case it becomes a noop if the cpu is offline.

Quoted 39 lines
arch/i386/kernel/cpuid.c renamed from arch/i386/kernel/cpuid.c~cpuid-msr-range-checking-fix +4 -2
@@ -133,10 +133,12 @@ static ssize_t cpuid_read(struct file *f
 static int cpuid_open(struct inode *inode, struct file *file)
 {
 	int cpu = iminor(file->f_dentry->d_inode);
-	struct cpuinfo_x86 *c = &(cpu_data)[cpu];
+	struct cpuinfo_x86 *c;
 
-	if (!cpu_online(cpu))
+	if (!cpu_possible(cpu) || !cpu_online(cpu))
 		return -ENXIO;	/* No such CPU */
+
+	c = &(cpu_data)[cpu];
 	if (c->cpuid_level < 0)
 		return -EIO;	/* CPUID not supported */
 

diff -puN arch/i386/kernel/msr.c~cpuid-msr-range-checking-fix arch/i386/kernel/msr.c

arch/i386/kernel/msr.c renamed from arch/i386/kernel/msr.c~cpuid-msr-range-checking-fix +6 -4
@@ -239,10 +239,12 @@ static ssize_t msr_write(struct file * f
 static int msr_open(struct inode *inode, struct file *file)
 {
   int cpu = iminor(file->f_dentry->d_inode);
-  struct cpuinfo_x86 *c = &(cpu_data)[cpu];
-  
-  if (!cpu_online(cpu))
-    return -ENXIO;		/* No such CPU */
+  struct cpuinfo_x86 *c;
+
+  if (!cpu_possible(cpu) || !cpu_online(cpu))
+    return -ENXIO;	/* No such CPU */
+
+  c = &(cpu_data)[cpu];
   if ( !cpu_has(c, X86_FEATURE_MSR) )
     return -EIO;		/* MSR not supported */
 

_

Signature · 1 line
Anyone who quotes me in their signature is an idiot -- Rusty Russell
#7 lore raw
In reply to Andrew Morton
Quoted 1 line · Andrew Morton

How about this?

Complicated *and* doesn't solve the original problem.

How about this?

Name: Fix i386/x86_64 cpuid/msr BUG() on Impossible CPUs
Status: Trivial

Matthieu Castet <[email protected]> pointed out that testing cpu_online(cpu) on a UP system goes BUG().

That's because you're never supposed to ask cpu_online() about a CPU which is >= NR_CPUS. msr and cpuid devices use the minor to indicate the CPU number. Oops.

Fix is to explicitly test cpu < NR_CPUS. Using cpu_online() is OK; although the CPU might go down before you actually read the file, that will simply cause junk to be returned.

arch/i386/kernel/cpuid.c +2 -2
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .7878-linux-2.6.6-bk6/arch/i386/kernel/cpuid.c .7878-linux-2.6.6-bk6.updated/arch/i386/kernel/cpuid.c
@@ -132,10 +132,10 @@ static ssize_t cpuid_read(struct file *f
 
 static int cpuid_open(struct inode *inode, struct file *file)
 {
-	int cpu = iminor(file->f_dentry->d_inode);
+	unsigned int cpu = iminor(file->f_dentry->d_inode);
 	struct cpuinfo_x86 *c = &(cpu_data)[cpu];
 
-	if (!cpu_online(cpu))
+	if (cpu >= NR_CPUS || !cpu_online(cpu))
 		return -ENXIO;	/* No such CPU */
 	if (c->cpuid_level < 0)
 		return -EIO;	/* CPUID not supported */
arch/i386/kernel/msr.c +2 -2
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .7878-linux-2.6.6-bk6/arch/i386/kernel/msr.c .7878-linux-2.6.6-bk6.updated/arch/i386/kernel/msr.c
@@ -238,10 +238,10 @@ static ssize_t msr_write(struct file * f
 
 static int msr_open(struct inode *inode, struct file *file)
 {
-  int cpu = iminor(file->f_dentry->d_inode);
+  unsigned int cpu = iminor(file->f_dentry->d_inode);
   struct cpuinfo_x86 *c = &(cpu_data)[cpu];
   
-  if (!cpu_online(cpu))
+  if (cpu >= NR_CPUS || !cpu_online(cpu))
     return -ENXIO;		/* No such CPU */
   if ( !cpu_has(c, X86_FEATURE_MSR) )
     return -EIO;		/* MSR not supported */
Signature · 1 line
Anyone who quotes me in their signature is an idiot -- Rusty Russell
#3 lore raw
In reply to Jeff Sipek
Quoted 1 line · Jeff

but on nosmp machine if the cpu!=0 this procude a BUG();

The line you refer to here is, I believe, in include/linux/cpumask.h:

  #ifdef CONFIG_SMP
    ...
  #else
    ...
  #define cpu_online(cpu)    ({ BUG_ON((cpu) != 0); 1; })
    ...
  #endif

Any thoughts on whether that BUG() is serving any useful purpose?

I'm of a mind to have the non-SMP case of cpu_online(cpu) simply respond true (1) hardcoded:

  #define cpu_online(cpu)    (1)

Or at least remove the BUG and have it respond true iff cpu == 0:

  #define cpu_online(cpu)    ((cpu) == 0)

While theoretically correct, that BUG() is, so far as I know, of no redeeming social value. And it wastes a few bytes of kernel text space each time cpu_online() is used on an SMP, and causes the occassional confusion, as in this case.

Signature · 3 lines
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <[email protected]> 1.650.933.1373
#8 lore raw
In reply to Andrew Morton
Quoted 7 lines · Andrew Morton
Quoted 4 lines · Rusty Russell

One way would be to do lock_cpu_hotplug() in cpuid_open() and introduce a cpuid_close() which would do unlock_cpu_hotplug(). Another would be to drop the check here, and fail the actual read. A final way is to do no checks, in which case it becomes a noop if the cpu is offline.

Too many options for me!

And this would be my preferred approach to the "cpu goes offline" race. Two patches in one mail is bad form, but it's late, they're short, and I'm lazy.

Compile tested.

Rusty.

Name: on_one_cpu function
Author: Rusty Russell
Status: Booted on 2.6.5

Similar to on_each_cpu, this implements on_one_cpu. For archs which don't do it natively, it's implemented in terms of smp_call_function().

include/linux/smp.h +23 -4
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .21074-linux-2.6.5/include/linux/smp.h .21074-linux-2.6.5.updated/include/linux/smp.h
@@ -8,6 +8,10 @@
 
 #include <linux/config.h>
 
+#define get_cpu()		({ preempt_disable(); smp_processor_id(); })
+#define put_cpu()		preempt_enable()
+#define put_cpu_no_resched()	preempt_enable_no_resched()
+
 #ifdef CONFIG_SMP
 
 #include <linux/preempt.h>
@@ -69,6 +73,24 @@ static inline int on_each_cpu(void (*fun
 	return ret;
 }
 
+#ifndef __HAVE_ARCH_ON_ONE_CPU
+extern int __on_one_cpu(unsigned cpu, void (*func)(void *info), void *info);
+static inline int on_one_cpu(unsigned cpu,
+			     void (*func)(void *info), void *info)
+{
+	int ret;
+
+	if (cpu == get_cpu()) {
+		func(info);
+		ret = 0;
+	} else
+		ret = __on_one_cpu(cpu, func, info);
+	put_cpu();
+	return ret;
+}
+#endif
+
+
 /*
  * True once the per process idle is forked
  */
@@ -123,6 +123,7 @@ void smp_prepare_boot_cpu(void);
 #define smp_threads_ready			1
 #define smp_call_function(func,info,retry,wait)	({ 0; })
 #define on_each_cpu(func,info,retry,wait)	({ func(info); 0; })
+#define on_one_cpu(cpu,func,info)		({ func(info); 0; })
 static inline void smp_send_reschedule(int cpu) { }
 #define num_booting_cpus()			1
 #define smp_prepare_boot_cpu()			do {} while (0)
@@ -107,8 +129,4 @@ static inline void smp_send_reschedule(i
 
 #endif /* !SMP */
 
-#define get_cpu()		({ preempt_disable(); smp_processor_id(); })
-#define put_cpu()		preempt_enable()
-#define put_cpu_no_resched()	preempt_enable_no_resched()
-
 #endif /* __LINUX_SMP_H */
kernel/sched.c +23 -1
diff -urpN --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal .21074-linux-2.6.5/kernel/sched.c .21074-linux-2.6.5.updated/kernel/sched.c
@@ -631,7 +631,29 @@ void kick_process(task_t *p)
 
 EXPORT_SYMBOL_GPL(kick_process);
 
-#endif
+#ifndef __HAVE_ARCH_ON_ONE_CPU
+struct which_cpu
+{
+	unsigned int cpu;
+	void (*func)(void *info);
+	void *info;
+};
+
+static void maybe_on_cpu(void *_which)
+{
+	struct which_cpu *which = _which;
+
+	if (smp_processor_id() == which->cpu)
+		which->func(which->info);
+}
+
+int __on_one_cpu(unsigned cpu, void (*func)(void *info), void *info)
+{
+	struct which_cpu which = { cpu, info };
+	return smp_call_function(maybe_on_cpu, &which, 1, 1);
+}
+#endif /* __HAVE_ARCH_ON_ONE_CPU */
+#endif /* CONFIG_SMP */
 
 /***
  * try_to_wake_up - wake up a thread

Name: Neater returns for i386/x86_64 cpuid/msr on Offline CPUs
Status: Untested
Depends: Misc/i386-cpu_online-BUG.patch.gz
Depends: Misc/on_one_cpu.patch.gz

You can open an MSR or CPUID file on a cpu, and then it can go down. You'll get junk/noop; nicer to return an error on read/write in this case.

Use on_one_cpu() as well. It's just easier to read.

arch/i386/kernel/cpuid.c +15 -28

76 lines not on this page · raw message

arch/i386/kernel/msr.c +17 -53

106 lines not on this page · raw message

Signature · 1 line
Anyone who quotes me in their signature is an idiot -- Rusty Russell
#4 lore raw
In reply to matthieu castet
Quoted 13 lines · matthieu castet

on monocpu machine (and maybe even on smp machine), when you try to
acces to a cpu that don't exist (/dev/cpu/1/cpuid), cpuid (msr) call cpu_online, but on nosmp machine if the cpu!=0 this procude a BUG(); So I add a check that verify if the cpu can exist before calling cpu_online.

Matthieu CASTET

[cpuid.patch text/x-patch (589 bytes)]
--- linux/arch/i386/kernel/cpuid.c 2004-05-18 20:47:05.000000000 +0200
+++ linux-2.6.5/arch/i386/kernel/cpuid.c 2004-04-04 05:36:12.000000000 +0200
@@ -135,7 +135,7 @@

Quoted 9 lines
    int cpu = iminor(file->f_dentry->d_inode);
    struct cpuinfo_x86 *c = &(cpu_data)[cpu];

- if (cpu >= num_possible_cpus() || !cpu_online(cpu))
+ if (!cpu_online(cpu))
return -ENXIO; /* No such CPU */
if ( c->cpuid_level < 0 )
return -EIO; /* CPUID not supported */

I think what you want here is

	if (!cpu_possible(cpu) || !cpu_online(cpu))
		return -ENXIO;

Like the below. Rusty agree?

arch/i386/kernel/cpuid.c renamed from arch/i386/kernel/cpuid.c~cpuid-msr-range-checking-fix +4 -2
@@ -133,10 +133,12 @@ static ssize_t cpuid_read(struct file *f
 static int cpuid_open(struct inode *inode, struct file *file)
 {
 	int cpu = iminor(file->f_dentry->d_inode);
-	struct cpuinfo_x86 *c = &(cpu_data)[cpu];
+	struct cpuinfo_x86 *c;
 
-	if (!cpu_online(cpu))
+	if (!cpu_possible(cpu) || !cpu_online(cpu))
 		return -ENXIO;	/* No such CPU */
+
+	c = &(cpu_data)[cpu];
 	if (c->cpuid_level < 0)
 		return -EIO;	/* CPUID not supported */
 
arch/i386/kernel/msr.c renamed from arch/i386/kernel/msr.c~cpuid-msr-range-checking-fix +6 -4
diff -puN arch/i386/kernel/msr.c~cpuid-msr-range-checking-fix arch/i386/kernel/msr.c
@@ -239,10 +239,12 @@ static ssize_t msr_write(struct file * f
 static int msr_open(struct inode *inode, struct file *file)
 {
   int cpu = iminor(file->f_dentry->d_inode);
-  struct cpuinfo_x86 *c = &(cpu_data)[cpu];
-  
-  if (!cpu_online(cpu))
-    return -ENXIO;		/* No such CPU */
+  struct cpuinfo_x86 *c;
+
+  if (!cpu_possible(cpu) || !cpu_online(cpu))
+    return -ENXIO;	/* No such CPU */
+
+  c = &(cpu_data)[cpu];
   if ( !cpu_has(c, X86_FEATURE_MSR) )
     return -EIO;		/* MSR not supported */
   

_

#6 lore raw
In reply to Rusty Russell
Quoted 10 lines · Rusty Russell
Quoted 4 lines · Andrew Morton

I think what you want here is

	if (!cpu_possible(cpu) || !cpu_online(cpu))
		return -ENXIO;

It works, but it's not really correct. cpu_possible() is correct, but cpu_online() might no longer be true by the time do_cpu_read() calls do_cpu_id().

mutter. Are we likely to see any ia32 or x86_64 hotplug-cpu machines?

Quoted 4 lines

One way would be to do lock_cpu_hotplug() in cpuid_open() and introduce a cpuid_close() which would do unlock_cpu_hotplug(). Another would be to drop the check here, and fail the actual read. A final way is to do no checks, in which case it becomes a noop if the cpu is offline.

Too many options for me!

Doing lock_cpu_hotplug() in open() would allow an app to take that lock permanently.

How about this?

 25-akpm/arch/i386/kernel/cpuid.c |   38 +++++++++++++++-----
 25-akpm/arch/i386/kernel/msr.c   |   74 ++++++++++++++++++++++++++++-----------
 2 files changed, 85 insertions(+), 27 deletions(-)
arch/i386/kernel/cpuid.c renamed from arch/i386/kernel/cpuid.c~cpuid-msr-range-checking-fix +30 -8
diff -puN arch/i386/kernel/cpuid.c~cpuid-msr-range-checking-fix arch/i386/kernel/cpuid.c
@@ -115,9 +115,19 @@ static ssize_t cpuid_read(struct file *f
 	size_t rv;
 	u32 reg = *ppos;
 	int cpu = iminor(file->f_dentry->d_inode);
+	ssize_t ret = 0;
 
-	if (count % 16)
-		return -EINVAL;	/* Invalid chunk size */
+	lock_cpu_hotplug();
+
+	if (!cpu_possible(cpu) || !cpu_online(cpu)) {
+		ret = -ENXIO;	/* No such CPU */
+		goto out;
+	}
+
+	if (count % 16) {
+		ret = -EINVAL;	/* Invalid chunk size */
+		goto out;
+	}
 
 	for (rv = 0; count; count -= 16) {
 		do_cpuid(cpu, reg, data);
@@ -127,20 +137,32 @@ static ssize_t cpuid_read(struct file *f
 		*ppos = reg++;
 	}
 
-	return ((char *)tmp) - buf;
+	ret = ((char *)tmp) - buf;
+out:
+	lock_cpu_hotplug();
+	return ret;
 }
 
 static int cpuid_open(struct inode *inode, struct file *file)
 {
 	int cpu = iminor(file->f_dentry->d_inode);
-	struct cpuinfo_x86 *c = &(cpu_data)[cpu];
+	struct cpuinfo_x86 *c;
+	int ret = 0;
 
-	if (!cpu_online(cpu))
-		return -ENXIO;	/* No such CPU */
+	lock_cpu_hotplug();
+
+	if (!cpu_possible(cpu) || !cpu_online(cpu)) {
+		ret = -ENXIO;	/* No such CPU */
+		goot out;
+	}
+
+	c = &(cpu_data)[cpu];
 	if (c->cpuid_level < 0)
-		return -EIO;	/* CPUID not supported */
+		ret = -EIO;	/* CPUID not supported */
 
-	return 0;
+out:
+	unlock_cpu_hotplug();
+	return ret;
 }
 
 /*
arch/i386/kernel/msr.c renamed from arch/i386/kernel/msr.c~cpuid-msr-range-checking-fix +55 -19

104 lines not on this page · raw message

_