Skip to contentSkip to conversation

cerberus on 2.4.17-rc2 UP

problem captured Manfred Spraul 2 participants 2 messages Jan 8, 2002 – Jan 9, 2002

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

Content-Type: multipart/mixed;
boundary="------------E7C7C7293155B3E004E01A0C"

This is a multi-part message in MIME format.
--------------E7C7C7293155B3E004E01A0C
Content-Type: text/plain; charset=us-ascii
Content-Transfer-Encoding: 7bit

Quoted 3 lines

Yes, I can generate it at will on two quite different IDE machines with the run-bash-shared-mapping script from
http://www.zip.com.au/~akpm/ext3-tools.tar.gz

Could you apply the attached patch and try to reproduce it? Enable CONFIG_DEBUG_SLAB.

The patch poisons all objects I could find that might have something to do with the bug. (all slab caches, struct request, struct page, struct filp, partially struct buffer_head).

My test box survives the run-bash_shared-mapping script (~30 min, 128 MB memory).

Thanks,
--
Manfred
--------------E7C7C7293155B3E004E01A0C
Content-Type: text/plain; charset=us-ascii;
name="patch-poison"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="patch-poison"

// $Header$
// Kernel Version:

//  VERSION = 2
//  PATCHLEVEL = 4
//  SUBLEVEL = 18
//  EXTRAVERSION = pre2
drivers/block/ll_rw_blk.c +8 -0
@@ -348,6 +348,8 @@
 		}
 		memset(rq, 0, sizeof(struct request));
 		rq->rq_status = RQ_INACTIVE;
+		poison_obj(&rq->elevator_sequence, sizeof(struct request)
+				-offsetof(struct request,elevator_sequence));
 		list_add(&rq->queue, &q->rq[i&1].free);
 		q->rq[i&1].count++;
 	}
@@ -428,8 +430,12 @@
 	if (!list_empty(&rl->free)) {
 		rq = blkdev_free_rq(&rl->free);
 		list_del(&rq->queue);
+		/* FIXME: clear or not clear? */
+		check_poison(&rq->elevator_sequence, sizeof(struct request)
+				- offsetof(struct request,elevator_sequence));
 		rl->count--;
 		rq->rq_status = RQ_ACTIVE;
+		rq->cmd = rw;
 		rq->special = NULL;
 		rq->q = q;
 	}
@@ -560,6 +566,8 @@
 	 */
 	if (q) {
 		list_add(&req->queue, &q->rq[rw].free);
+		poison_obj(&req->elevator_sequence, sizeof(struct request)
+				-offsetof(struct request,elevator_sequence));
 		if (++q->rq[rw].count >= batch_requests && waitqueue_active(&q->wait_for_request))
 			wake_up(&q->wait_for_request);
 	}
fs/file_table.c +6 -0
@@ -39,6 +39,8 @@
 	used_one:
 		f = list_entry(free_list.next, struct file, f_list);
 		list_del(&f->f_list);
+		check_poison (&f->f_dentry, sizeof(struct file) -
+				offsetof(struct file, f_dentry));
 		files_stat.nr_free_files--;
 	new_one:
 		memset(f, 0, sizeof(*f));
@@ -118,6 +120,8 @@
 		file->f_dentry = NULL;
 		file->f_vfsmnt = NULL;
 		list_del(&file->f_list);
+		poison_obj(&file->f_dentry, sizeof(struct file) -
+				offsetof(struct file, f_dentry));
 		list_add(&file->f_list, &free_list);
 		files_stat.nr_free_files++;
 		file_list_unlock();
@@ -147,6 +151,8 @@
 		file_list_lock();
 		list_del(&file->f_list);
 		list_add(&file->f_list, &free_list);
+		poison_obj(&file->f_dentry, sizeof(struct file) -
+				offsetof(struct file, f_dentry));
 		files_stat.nr_free_files++;
 		file_list_unlock();
 	}
include/linux/slab.h +34 -0
@@ -78,6 +78,40 @@
 extern kmem_cache_t	*fs_cachep;
 extern kmem_cache_t	*sigact_cachep;
 
+
+#ifdef CONFIG_DEBUG_SLAB
+extern void __check_poison(void *obj, int size, char *file, int line);
+
+#define check_and_clear_poison(obj, size) \
+		do { \
+			__check_poison(obj, size, __FILE__, __LINE__); \
+			memset(obj, 0, size); \
+		} while(0)
+
+#define check_poison(obj, size) \
+		do { \
+			__check_poison(obj, size, __FILE__, __LINE__); \
+			memset(obj, 0x2C, size); \
+		} while(0)
+
+#define poison_obj(obj, size) \
+			memset(obj, 0xe3, size); \
+
+#else
+static inline void check_and_clear_poison(void *obj, int size)
+{
+	memset(obj, 0, size);
+}
+static inline void check_poison(void *obj, int size)
+{
+	/* nop */
+}
+static inline void poison_obj(void *obj, int size)
+{
+	/* nop */
+}
+#define 
+#endif
 #endif	/* __KERNEL__ */
 
 #endif	/* _LINUX_SLAB_H */
mm/slab.c +40 -1

75 lines not on this page · raw message

mm/page_alloc.c +30 -1
@@ -89,7 +89,16 @@
 	if (current->flags & PF_FREE_PAGES)
 		goto local_freelist;
  back_local_freelist:
-
+#ifdef CONFIG_DEBUG_SLAB
+	page->mapping = (void*)0xdeadbeef;
+	page->index = 0xbaadc0de;
+	page->next_hash = (void*)0xbeefdead;
+	atomic_set(&page->count,1);
+	page->lru.next = (void*)0xbaadf00d;
+	page->lru.prev = (void*)0xf00dbaad;
+	page->pprev_hash = (void*)0xdeadbeef;
+	page->buffers = (void*)0xdeadbeef;
+#endif
 	zone = page->zone;
 
 	mask = (~0UL) << order;
@@ -200,6 +209,26 @@
 			page = expand(zone, page, index, order, curr_order, area);
 			spin_unlock_irqrestore(&zone->lock, flags);
 
+#ifdef CONFIG_DEBUG_SLAB
+			if (page->mapping != (void*)0xdeadbeef)
+				printk(KERN_ERR"got mapping %p.\n", page->mapping);
+			if (page->index != 0xbaadc0de)
+				printk(KERN_ERR "got index %lxh.\n", page->index);
+			if (page->next_hash != (void*)0xbeefdead)
+				printk(KERN_ERR "got next_hash %p.\n", page->next_hash);
+			if (atomic_read(&page->count) != 1)
+				printk(KERN_ERR "bad page count %d.\n", atomic_read(&page->count));
+			if (page->lru.next != (void*)0xbaadf00d)
+				printk(KERN_ERR" bad lru_next %p.\n", page->lru.next);
+			if (page->lru.prev != (void*)0xf00dbaad)
+				printk(KERN_ERR" bad lru_prev %p.\n", page->lru.prev);
+			if (page->pprev_hash != (void*)0xdeadbeef)
+				printk(KERN_ERR" bad pprev_hash %p.\n", page->pprev_hash);
+			if (page->buffers != (void*)0xdeadbeef)
+				printk(KERN_ERR" bad page->buffers %p.\n", page->buffers); 
+			page->mapping = NULL;
+			page->buffers = NULL; 
+#endif
 			set_page_count(page, 1);
 			if (BAD_RANGE(zone,page))
 				BUG();
fs/buffer.c +5 -0
@@ -1197,6 +1197,7 @@
 		bh->b_dev = B_FREE;
 		bh->b_blocknr = -1;
 		bh->b_this_page = NULL;
+		bh->b_size = 0xbaad;
 
 		nr_unused_buffer_heads++;
 		bh->b_next_free = unused_list;
@@ -1227,6 +1228,8 @@
 		unused_list = bh->b_next_free;
 		nr_unused_buffer_heads--;
 		spin_unlock(&unused_list_lock);
+		if (bh->b_size != 0xbaad)
+			printk(KERN_ERR "wrong size %lxh.\n", bh->b_size);
 		return bh;
 	}
 	spin_unlock(&unused_list_lock);
@@ -1251,6 +1254,8 @@
 			unused_list = bh->b_next_free;
 			nr_unused_buffer_heads--;
 			spin_unlock(&unused_list_lock);
+			if (bh->b_size != 0xbaad)
+				printk(KERN_ERR "wrong size %lxh.\n", bh->b_size);
 			return bh;
 		}
 		spin_unlock(&unused_list_lock);

--------------E7C7C7293155B3E004E01A0C--

#1 lore raw
Quoted 6 lines · Manfred Spraul
Quoted 4 lines

Yes, I can generate it at will on two quite different IDE machines with the run-bash-shared-mapping script from
http://www.zip.com.au/~akpm/ext3-tools.tar.gz

Could you apply the attached patch and try to reproduce it?

Nice patch.

Quoted 9 lines

Enable CONFIG_DEBUG_SLAB.

The patch poisons all objects I could find that might have something to do with the bug. (all slab caches, struct request, struct page, struct filp, partially struct buffer_head).

My test box survives the run-bash_shared-mapping script (~30 min, 128 MB memory).

Mine survives only a few minutes. Once it only lasted a second. That's with mem=64m. It lasts much, much longer with more memory.

The patch, alas, sheds no light. I'll delve into it fairly soon, I expect.

EXT3-fs: recovery complete.
EXT3-fs: mounted filesystem with ordered data mode.
VFS: Mounted root (ext3 filesystem) readonly.
Freeing unused kernel memory: 212k freed
end_request: buffer-list destroyed
hda6: bad access: block=86256, count=-8
end_request: I/O error, dev 03:06 (hda), sector 86256
hda: timeout waiting for DMA
ide_dmaproc: chipset supported ide_dma_timeout func only: 14
hda: status error: status=0x58 { DriveReady SeekComplete DataRequest }
hda: drive not ready for command
hda: lost interrupt

and:

end_request: buffer-list destroyed
hda6: bad access: block=93608, count=-8
end_request: I/O error, dev 03:06 (hda), sector 93608
hda6: bad access: block=93616, count=-16
end_request: I/O error, dev 03:06 (hda), sector 93616
hda6: bad access: block=93624, count=-24
end_request: I/O error, dev 03:06 (hda), sector 93624
hda6: bad access: block=93632, count=-32
end_request: I/O error, dev 03:06 (hda), sector 93632
hda6: bad access: block=93640, count=-40
end_request: I/O error, dev 03:06 (hda), sector 93640
hda6: bad access: block=93648, count=-48
end_request: I/O error, dev 03:06 (hda), sector 93648
hda6: bad access: block=93656, count=-56
end_request: I/O error, dev 03:06 (hda), sector 93656
hda6: bad access: block=93664, count=-64
end_request: I/O error, dev 03:06 (hda), sector 93664
hda6: bad access: block=93672, count=-72
end_request: I/O error, dev 03:06 (hda), sector 93672
hda6: bad access: block=93680, count=-80
end_request: I/O error, dev 03:06 (hda), sector 93680
hda6: bad access: block=93688, count=-88
end_request: I/O error, dev 03:06 (hda), sector 93688
hda6: bad access: block=93696, count=-96
end_request: I/O error, dev 03:06 (hda), sector 93696
hda6: bad access: block=93704, count=-104
end_request: I/O error, dev 03:06 (hda), sector 93704
hda: timeout waiting for DMA
ide_dmaproc: chipset supported ide_dma_timeout func only: 14
hda: status error: status=0x58 { DriveReady SeekComplete DataRequest }
hda: drive not ready for command
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt
hda: lost interrupt

hmm.. hda6 is the root filesystem. The test was hitting hda8 and hda5(swap). The only activity happening on hda6 would be a bit of pagein, maybe syslog. hmm.

Always hda6:

end_request: buffer-list destroyed
hda6: bad access: block=90704, count=-8
end_request: I/O error, dev 03:06 (hda), sector 90704
hda6: bad access: block=90712, count=-16
end_request: I/O error, dev 03:06 (hda), sector 90712
hda6: bad access: block=90720, count=-24
end_request: I/O error, dev 03:06 (hda), sector 90720
hda6: bad access: block=90728, count=-32

Interestingly, 2.4.13-ac8 doesn't fail. Well, it eventually takes oopses in do_IRQ()'s get_current() - %cr2 has value 0x4017a000.

That kernel has the new IDE drivers, but I've seen the problem with Andre's latest patches on PIIX, on VIA, and there are reports of it on SCSI. And "buffer-list destroyed" is always the first message. It doesn't feel like a driver problem. I'll go do a binary search through some kernel revs.

-