cerberus on 2.4.17-rc2 UP
Subsystems (5)
Content-Type: multipart/mixed;
boundary="------------E7C7C7293155B3E004E01A0C"
This is a multi-part message in MIME format.
--------------E7C7C7293155B3E004E01A0C
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Quoted 3 lines
Yes, I can generate it at will on two quite different IDE machines with the run-bash-shared-mapping script from
http://www.zip.com.au/~akpm/ext3-tools.tar.gz
Could you apply the attached patch and try to reproduce it? Enable CONFIG_DEBUG_SLAB.
The patch poisons all objects I could find that might have something to do with the bug. (all slab caches, struct request, struct page, struct filp, partially struct buffer_head).
My test box survives the run-bash_shared-mapping script (~30 min, 128 MB memory).
Thanks,
--
Manfred
--------------E7C7C7293155B3E004E01A0C
Content-Type: text/plain; charset=us-ascii;
name="patch-poison"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="patch-poison"
// $Header$
// Kernel Version:
// VERSION = 2 // PATCHLEVEL = 4 // SUBLEVEL = 18 // EXTRAVERSION = pre2
drivers/block/ll_rw_blk.c +8 -0
@@ -348,6 +348,8 @@ }memset (rq,0 ,sizeof (struct request )); rq->rq_status =RQ_INACTIVE ; +poison_obj (&rq->elevator_sequence,sizeof (struct request ) + -offsetof(struct request ,elevator_sequence));list_add (&rq->queue, &q->rq[i&1 ].free); q->rq[i&1 ].count++; }
@@ -428,8 +430,12 @@if (!list_empty (&rl->free)) { rq =blkdev_free_rq (&rl->free);list_del (&rq->queue); +/* FIXME: clear or not clear? */ +check_poison (&rq->elevator_sequence,sizeof (struct request ) + - offsetof(struct request ,elevator_sequence)); rl->count--; rq->rq_status =RQ_ACTIVE ; + rq->cmd = rw; rq->special =NULL ; rq->q = q; }
@@ -560,6 +566,8 @@*/ if (q) {list_add (&req->queue, &q->rq[rw].free); +poison_obj (&req->elevator_sequence,sizeof (struct request ) + -offsetof(struct request ,elevator_sequence));if (++q->rq[rw].count >= batch_requests &&waitqueue_active (&q->wait_for_request))wake_up (&q->wait_for_request); }
fs/file_table.c +6 -0
@@ -39,6 +39,8 @@ used_one: f = list_entry(free_list.next, struct file, f_list);list_del (&f->f_list); +check_poison (&f->f_dentry,sizeof (struct file ) - + offsetof(struct file , f_dentry)); files_stat.nr_free_files--; new_one:memset (f,0 ,sizeof (*f));
@@ -118,6 +120,8 @@ file->f_dentry =NULL ; file->f_vfsmnt =NULL ;list_del (&file->f_list); +poison_obj (&file->f_dentry,sizeof (struct file ) - + offsetof(struct file , f_dentry));list_add (&file->f_list, &free_list); files_stat.nr_free_files++;file_list_unlock ();
@@ -147,6 +151,8 @@file_list_lock ();list_del (&file->f_list);list_add (&file->f_list, &free_list); +poison_obj (&file->f_dentry,sizeof (struct file ) - + offsetof(struct file , f_dentry)); files_stat.nr_free_files++;file_list_unlock (); }
include/linux/slab.h +34 -0
@@ -78,6 +78,40 @@extern kmem_cache_t *fs_cachep;extern kmem_cache_t *sigact_cachep; + +#ifdef CONFIG_DEBUG_SLAB +extern void __check_poison (void *obj,int size,char *file,int line); + +#define check_and_clear_poison (obj, size) \ + do { \ + __check_poison(obj, size, __FILE__, __LINE__); \ + memset(obj, 0, size); \ + } while(0) + +#define check_poison (obj, size) \ + do { \ + __check_poison(obj, size, __FILE__, __LINE__); \ + memset(obj, 0x2C, size); \ + } while(0) + +#define poison_obj (obj, size) \ + memset(obj, 0xe3, size); \ + +#else +static inline void check_and_clear_poison (void *obj,int size) +{ +memset (obj,0 , size); +} +static inline void check_poison (void *obj,int size) +{ +/* nop */ +} +static inline void poison_obj (void *obj,int size) +{ +/* nop */ +} +#define +#endif #endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */
mm/slab.c +40 -1
75 lines not on this page · raw message
mm/page_alloc.c +30 -1
@@ -89,7 +89,16 @@if (current->flags &PF_FREE_PAGES ) goto local_freelist; back_local_freelist: - +#ifdef CONFIG_DEBUG_SLAB + page->mapping = (void *)0xdeadbeef ; + page->index =0xbaadc0de ; + page->next_hash = (void *)0xbeefdead ; +atomic_set (&page->count,1 ); + page->lru.next = (void *)0xbaadf00d ; + page->lru.prev = (void *)0xf00dbaad ; + page->pprev_hash = (void *)0xdeadbeef ; + page->buffers = (void *)0xdeadbeef ; +#endif zone = page->zone; mask = (~0UL ) << order;
@@ -200,6 +209,26 @@ page =expand (zone, page, index, order, curr_order, area);spin_unlock_irqrestore (&zone->lock, flags); +#ifdef CONFIG_DEBUG_SLAB +if (page->mapping != (void *)0xdeadbeef ) +printk (KERN_ERR "got mapping %p.\n" , page->mapping); +if (page->index !=0xbaadc0de ) +printk (KERN_ERR "got index %lxh.\n" , page->index); +if (page->next_hash != (void *)0xbeefdead ) +printk (KERN_ERR "got next_hash %p.\n" , page->next_hash); +if (atomic_read (&page->count) !=1 ) +printk (KERN_ERR "bad page count %d.\n" ,atomic_read (&page->count)); +if (page->lru.next != (void *)0xbaadf00d ) +printk (KERN_ERR " bad lru_next %p.\n" , page->lru.next); +if (page->lru.prev != (void *)0xf00dbaad ) +printk (KERN_ERR " bad lru_prev %p.\n" , page->lru.prev); +if (page->pprev_hash != (void *)0xdeadbeef ) +printk (KERN_ERR " bad pprev_hash %p.\n" , page->pprev_hash); +if (page->buffers != (void *)0xdeadbeef ) +printk (KERN_ERR " bad page->buffers %p.\n" , page->buffers); + page->mapping =NULL ; + page->buffers =NULL ; +#endif set_page_count (page,1 );if (BAD_RANGE (zone,page))BUG ();
fs/buffer.c +5 -0
@@ -1197,6 +1197,7 @@ bh->b_dev =B_FREE ; bh->b_blocknr =-1 ; bh->b_this_page =NULL ; + bh->b_size =0xbaad ; nr_unused_buffer_heads++; bh->b_next_free = unused_list;
@@ -1227,6 +1228,8 @@ unused_list = bh->b_next_free; nr_unused_buffer_heads--;spin_unlock (&unused_list_lock); +if (bh->b_size !=0xbaad ) +printk (KERN_ERR "wrong size %lxh.\n" , bh->b_size);return bh; }spin_unlock (&unused_list_lock);
@@ -1251,6 +1254,8 @@ unused_list = bh->b_next_free; nr_unused_buffer_heads--;spin_unlock (&unused_list_lock); +if (bh->b_size !=0xbaad ) +printk (KERN_ERR "wrong size %lxh.\n" , bh->b_size);return bh; }spin_unlock (&unused_list_lock);
--------------E7C7C7293155B3E004E01A0C--
Quoted 6 lines · Manfred Spraul
Quoted 4 lines
Yes, I can generate it at will on two quite different IDE machines with the run-bash-shared-mapping script from
http://www.zip.com.au/~akpm/ext3-tools.tar.gzCould you apply the attached patch and try to reproduce it?
Nice patch.
Quoted 9 lines
Enable
CONFIG_DEBUG_SLAB.The patch poisons all objects I could find that might have something to do with the bug. (all slab caches, struct request, struct page, struct filp, partially struct
buffer_head).My test box survives the run-bash_shared-mapping script (~30 min, 128 MB memory).
Mine survives only a few minutes. Once it only lasted a second. That's with mem=64m. It lasts much, much longer with more memory.
The patch, alas, sheds no light. I'll delve into it fairly soon, I expect.
EXT3-fs: recovery complete.
EXT3-fs: mounted filesystem with ordered data mode.
VFS: Mounted root (ext3 filesystem) readonly.
Freeing unused kernel memory: 212k freedend_request: buffer-list destroyed
hda6: bad access: block=86256, count=-8end_request: I/O error, dev 03:06 (hda), sector 86256
hda: timeout waiting for DMAide_dmaproc: chipset supported ide_dma_timeout func only: 14
hda: status error: status=0x58 { DriveReady SeekComplete DataRequest }
hda: drive not ready for command
hda: lost interrupt
and:
end_request: buffer-list destroyed
hda6: bad access: block=93608, count=-8end_request: I/O error, dev 03:06 (hda), sector 93608
hda6: bad access: block=93616, count=-16end_request: I/O error, dev 03:06 (hda), sector 93616
hda6: bad access: block=93624, count=-24end_request: I/O error, dev 03:06 (hda), sector 93624
hda6: bad access: block=93632, count=-32end_request: I/O error, dev 03:06 (hda), sector 93632
hda6: bad access: block=93640, count=-40end_request: I/O error, dev 03:06 (hda), sector 93640
hda6: bad access: block=93648, count=-48end_request: I/O error, dev 03:06 (hda), sector 93648
hda6: bad access: block=93656, count=-56end_request: I/O error, dev 03:06 (hda), sector 93656
hda6: bad access: block=93664, count=-64end_request: I/O error, dev 03:06 (hda), sector 93664
hda6: bad access: block=93672, count=-72end_request: I/O error, dev 03:06 (hda), sector 93672
hda6: bad access: block=93680, count=-80end_request: I/O error, dev 03:06 (hda), sector 93680
hda6: bad access: block=93688, count=-88end_request: I/O error, dev 03:06 (hda), sector 93688
hda6: bad access: block=93696, count=-96end_request: I/O error, dev 03:06 (hda), sector 93696
hda6: bad access: block=93704, count=-104end_request: I/O error, dev 03:06 (hda), sector 93704
hda: timeout waiting for DMAide_dmaproc: chipset supported ide_dma_timeout func only: 14
hda: status error: status=0x58 { DriveReady SeekComplete DataRequest }
hda: drive not ready for command
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hmm.. hda6 is the root filesystem. The test was hitting hda8 and hda5(swap). The only activity happening on hda6 would be a bit of pagein, maybe syslog. hmm.
Always hda6:
end_request: buffer-list destroyed
hda6: bad access: block=90704, count=-8end_request: I/O error, dev 03:06 (hda), sector 90704
hda6: bad access: block=90712, count=-16end_request: I/O error, dev 03:06 (hda), sector 90712
hda6: bad access: block=90720, count=-24end_request: I/O error, dev 03:06 (hda), sector 90720
hda6: bad access: block=90728, count=-32
Interestingly, 2.4.13-ac8 doesn't fail. Well, it eventually takes oopses in do_IRQ()'s get_current() - %cr2 has value 0x4017a000.
That kernel has the new IDE drivers, but I've seen the problem with Andre's latest patches on PIIX, on VIA, and there are reports of it on SCSI. And "buffer-list destroyed" is always the first message. It doesn't feel like a driver problem. I'll go do a binary search through some kernel revs.
-