memory corruptor in .18rc1-git
Subsystems (4)
Three times in the last week, I've had a box running the -git-du-jour spontaneously reboot. It just happened again, this time I had a serial console hooked up, but it rebooted before transferring much data. The one thing it did spew however was "List corruption. prev->ne", which came from the patch below which I had in my tree.
(The latter half is likely irrelevant, and came from chasing a different bug)
Things in common at all three times it happened.. reading email, and listening to oggs with rhythmbox. Another ALSA bug maybe ?
I've up'd the speed of the serial console, in the hope that more chars make it over the wire before reboot should this happen again.
Dave
include/linux/list.h renamed from include/linux/list.h~ +22 -0
@@ -5,7 +5,9 @@#include <linux/stddef.h> #include <linux/prefetch.h> +#include <linux/kernel.h> #include <asm/system.h> +#include <asm/bug.h> /* * These are non-NULL pointers that will result in page faults
@@ -52,6 +52,16 @@ static inline void __list_add(struct lis struct list_head *prev, struct list_head *next) { +if (next->prev != prev) { +printk ("List corruption. next->prev should be %p, but was %p\n" , + prev, next->prev); +BUG (); + } +if (prev->next != next) { +printk ("List corruption. prev->next should be %p, but was %p\n" , + next, prev->next); +BUG (); + } next->prev = new; new->next = next; new->prev = prev;
@@ -162,6 +162,16 @@ static inline void __list_del(struct lis*/ static inline void list_del (struct list_head *entry) { +if (entry->prev->next != entry) { +printk ("List corruption. prev->next should be %p, but was %p\n" , + entry, entry->prev->next); +BUG (); + } +if (entry->next->prev != entry) { +printk ("List corruption. next->prev should be %p, but was %p\n" , + entry, entry->next->prev); +BUG (); + }__list_del (entry->prev, entry->next); entry->next =LIST_POISON1 ; entry->prev =LIST_POISON2 ;
It _may_ give more information about exactly which wait-queue (or rather, what the poll function that was related to that wait-queue) seems to be corrupted.
Linus
---
fs/select.c +3 -1
@@ -23,6 +23,7 @@ #include <linux/personality.h> /* for ST#include <linux/file.h> #include <linux/fs.h> #include <linux/rcupdate.h> +#include <linux/kallsyms.h> #include <asm/uaccess.h>
@@ -65,7 +66,8 @@ EXPORT_SYMBOL(poll_initwait);static void free_poll_entry (struct poll_table_entry *entry) { -remove_wait_queue (entry->wait_address,&entry->wait); +if (remove_wait_queue (entry->wait_address,&entry->wait) <0 ) +print_symbol ("bad poll-entry for %s" , (unsigned long ) entry->filp->f_op->poll);fput (entry->filp); }
include/linux/wait.h +1 -1
@@ -115,7 +115,7 @@ #define is_sync_wait(wait) (!(wait) || (extern void FASTCALL (add_wait_queue (wait_queue_head_t *q,wait_queue_t * wait));extern void FASTCALL (add_wait_queue_exclusive (wait_queue_head_t *q,wait_queue_t * wait)); -extern void FASTCALL (remove_wait_queue (wait_queue_head_t *q,wait_queue_t * wait)); +extern int FASTCALL (remove_wait_queue (wait_queue_head_t *q,wait_queue_t * wait));static inline void __add_wait_queue (wait_queue_head_t *head,wait_queue_t *new) {
kernel/wait.c +24 -2
@@ -33,13 +33,35 @@ void fastcall add_wait_queue_exclusive(w }EXPORT_SYMBOL (add_wait_queue_exclusive); -void fastcall remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) +int fastcall remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) {unsigned long flags; +struct list_head *list; +int seen, retval;spin_lock_irqsave (&q->lock, flags); -__remove_wait_queue (q, wait); + list = &q->task_list; + seen =0 ; + retval =-1 ; + +do { +struct list_head *next; +if (list == &wait->task_list) + seen++; + next = list->next; +if (next->prev != list) { + seen +=2 ; +break ; + } + list = next; + }while (list != &q->task_list); + +if (seen ==1 ) { +__remove_wait_queue (q, wait); + retval =0 ; + }spin_unlock_irqrestore (&q->lock, flags); +return retval; }EXPORT_SYMBOL (remove_wait_queue);
Signature · 1 line
http://www.codemonkey.org.uk
Quoted 7 lines · On Thu, 13 Jul 2006 18:13:30 -0400 Dave Jones <[email protected]> wrote:
Three times in the last week, I've had a box running the -git-du-jour spontaneously reboot. It just happened again, this time I had a serial console hooked up, but it rebooted before transferring much data. The one thing it did spew however was "List corruption.
prev->ne", which came from the patch below which I had in my tree.
(The latter half is likely irrelevant, and came from chasing a different bug)
Quoted 6 lines
Things in common at all three times it happened.. reading email, and listening to oggs with rhythmbox. Another ALSA bug maybe ?
I've up'd the speed of the serial console, in the hope that more chars make it over the wire before reboot should this happen again.
Are you using SMP? We have a known slab locking bug.
There have been a couple of slab.c patches committed today, but neither of them appear to actually fix the bug.
The below should fix it, and testing this (disable lockdep) would be useful.
It's going to take a bit of work to unpickle it all now.
mm/slab.c renamed from mm/slab.c~revert-slabc-lockdep-locking-change +0 -9
@@ -3100,16 +3100,7 @@ static void free_block(struct kmem_cacheif (slabp->inuse ==0 ) {if (l3->free_objects > l3->free_limit) { l3->free_objects -= cachep->num; -/* -* It is safe to drop the lock. The slab is -* no longer linked to the cache. cachep -* cannot disappear - we are using it and -* all destruction of caches must be -* serialized properly by the user. -*/ -spin_unlock (&l3->list_lock);slab_destroy (cachep, slabp); -spin_lock (&l3->list_lock); }else {list_add (&slabp->list, &l3->slabs_free); }
_
Quoted 4 lines · Andrew Morton
Quoted 2 lines
I've up'd the speed of the serial console, in the hope that more chars make it over the wire before reboot should this happen again.
Are you using SMP? We have a known slab locking bug.
Yes, dual EM64T's with HT.
Quoted 5 lines
There have been a couple of
slab.cpatches committed today, but neither of them appear to actually fix the bug.The below should fix it, and testing this (disable lockdep) would be useful.
I can give it a shot, but as it takes a while for this to manifest, I may not be able to say for certain whether it fixes it or not.
Dave
Signature · 1 line
http://www.codemonkey.org.uk
Quoted 9 lines · Dave Jones
Quoted 4 lines · Andrew Morton
Quoted 2 lines
I've up'd the speed of the serial console, in the hope that more chars make it over the wire before reboot should this happen again.
Are you using SMP? We have a known slab locking bug.
Yes, dual EM64T's with HT.
Quoted 8 lines
Quoted 5 lines
There have been a couple of
slab.cpatches committed today, but neither of them appear to actually fix the bug.The below should fix it, and testing this (disable lockdep) would be useful.
I can give it a shot, but as it takes a while for this to manifest, I may not be able to say for certain whether it fixes it or not.
Then you might consider slightly changing the debug messages, because they are identical in list_add and list_del. Having a way to differenciate between the two functions might give one more indication.
Quoted 1 line
Dave
Willy
Quoted 6 lines · Willy Tarreau
Quoted 2 lines
I can give it a shot, but as it takes a while for this to manifest, I may not be able to say for certain whether it fixes it or not.
Then you might consider slightly changing the debug messages, because they are identical in
list_addandlist_del. Having a way to differenciate between the two functions might give one more indication.
BUG() gives a line number.
Dave
Signature · 1 line
http://www.codemonkey.org.uk
Quoted 10 lines · Dave Jones
Quoted 6 lines · Willy Tarreau
Quoted 2 lines
I can give it a shot, but as it takes a while for this to manifest, I may not be able to say for certain whether it fixes it or not.
Then you might consider slightly changing the debug messages, because they are identical in
list_addandlist_del. Having a way to differenciate between the two functions might give one more indication.
BUG()gives a line number.
oops! sorry, I did not notice it just after the printk(). Next time I will not post before coffee :-)
Quoted 1 line
Dave
Willy
Quoted 8 lines · Willy Tarreau
Quoted 5 lines
Quoted 3 lines
Then you might consider slightly changing the debug messages, because they are identical in
list_addandlist_del. Having a way to differenciate between the two functions might give one more indication.
BUG()gives a line number.oops! sorry, I did not notice it just after the
printk(). Next time I will not post before coffee :-)
Though in the case I hit this morning, the text from the BUG() wouldn't have made it over serial. But then, unless the first few chars were different, not much else would have either. It didn't even complete the printk.
For most cases though, it should suffice. It gives enough clues that 'something stomped on memory' in cases like I saw, and in other cases where the box still survives, it'll be possible to get the full context from dmesg. For info, here's a cleaned up variant of the patch with some suggestions from Linus. Andrew also merged an earlier variant into -mm.
Dave
include/linux/list.h +9 -0
@@ -57,10 +57,15 @@ static inline void __list_add(struct lis* Insert a new entry after the specified head. * This is good for implementing stacks. */ +#ifndef CONFIG_DEBUG_LIST static inline void list_add (struct list_head *new,struct list_head *head) {__list_add (new, head, head->next); } +#else +extern void list_add (struct list_head *new,struct list_head *head); +#endif +/** * list_add_tail - add a new entry
@@ -153,12 +158,16 @@ static inline void __list_del(struct lis* Note: list_empty on entry does not return true after this, the entry is * in an undefined state. */ +#ifndef CONFIG_DEBUG_LIST static inline void list_del (struct list_head *entry) {__list_del (entry->prev, entry->next); entry->next =LIST_POISON1 ; entry->prev =LIST_POISON2 ; } +#else +extern void list_del (struct list_head *entry); +#endif /** * list_del_rcu - deletes entry from list without re-initialization
lib/Kconfig.debug +9 -0
@@ -311,6 +311,15 @@ config DEBUG_VM If unsure, say N. +config DEBUG_LIST + bool "Debug linked list manipulation" + depends on DEBUG_KERNEL + help + Enable this to turn on extended checks in the linked-list + walking routines. + + If unsure, say N. + config FRAME_POINTER bool "Compile the kernel with frame pointers" depends on DEBUG_KERNEL && (X86 || CRIS || M68K || M68KNOMMU || FRV || UML || S390)
lib/Makefile +1 -0
@@ -28,6 +28,7 @@ lib-$(CONFIG_GENERIC_HWEIGHT) += hweight obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o obj-$(CONFIG_PLIST) += plist.o obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o +obj-$(CONFIG_DEBUG_LIST) += list_debug.o ifneq ($(CONFIG_HAVE_DEC_LOCK),y) lib-y += dec_and_lock.o
lib/list_debug.c new file +60 -0
60 lines not on this page · raw message
Signature · 1 line
http://www.codemonkey.org.uk