xfs crash, kernel 2.6.29

We had a new crash yesterday, server uptime maybe 20 hours.

This is 2.6.29-xen-r4 from
http://code.google.com/p/gentoo-xen-kernel/downloads/list

@Eric/Andrew: do we have a xfs or a xen patches problem?

Christian

------------[ cut here ]------------
Kernel BUG at ffffffff8045281d [verbose debug info unavailable]

invalid opcode: 0000 [#1] SMP
last sysfs
file: /sys/devices/xen/pci-0/pci0000:00/0000:00:00.0/host3/target3:0:6/3:0:6:0/rev CPU 3

Modules linked in: nfsd
Pid: 227, comm: kswapd0 Not tainted 2.6.29-xen-r4 #13
RIP: e030:[<ffffffff8045281d>]  [<ffffffff8045281d>] 
radix_tree_tag_set+0x9d/0xb0
RSP: e02b:ffff8800ff97bce0  EFLAGS: 00010246
RAX: 000000000000002d RBX: 0000000000000000 RCX: ffff8800c13ceba8
RDX: 0000000000000000 RSI: 000000000022df71 RDI: ffff8800fe866e40
RBP: ffff8800fe866e00 R08: 000000000000002d R09: 000000000000000c
R10: 0000000000000000 R11: 0000000000000003 R12: ffff8800fec59c00
R13: ffff8800067f7938 R14: 0000000000000080 R15: 000000000004828f
FS:  00007f0617e8e6f0(0000) GS:ffff8800010061c0(0000) knlGS:0000000000000000
CS:  e033 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f187e14a110 CR3: 00000000fdcbd000 CR4: 0000000000002620
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process kswapd0 (pid: 227, threadinfo ffff8800ff97a000, task ffff8800ffb50540)
Stack:
ffff8800067f7840 ffffffff80422100 000000000004828f ffff8800067f7840
ffff8800067f79c0 ffff8800ff97bd80 0000000000000012 ffffffff8041229b
ffff8800067f79c0 ffff8800067f79c0 ffff8800067f79c0 ffffffff80421146
Call Trace:
[<ffffffff80422100>] ? xfs_inode_set_reclaim_tag+0x80/0xd0
[<ffffffff8041229b>] ? xfs_reclaim+0x6b/0xe0
[<ffffffff80421146>] ? xfs_fs_destroy_inode+0x36/0x60
[<ffffffff802a8c12>] ? dispose_list+0xa2/0x150
[<ffffffff802a8e9f>] ? shrink_icache_memory+0x1df/0x310
[<ffffffff8026da44>] ? shrink_slab+0x124/0x180
[<ffffffff8026e24d>] ? kswapd+0x3bd/0x600
[<ffffffff8026b700>] ? isolate_pages_global+0x0/0x280
[<ffffffff80245670>] ? autoremove_wake_function+0x0/0x30
[<ffffffff802242fb>] ? __wake_up_common+0x5b/0x90
[<ffffffff8026de90>] ? kswapd+0x0/0x600
[<ffffffff802451eb>] ? kthread+0x4b/0x80
[<ffffffff8020b66a>] ? child_rip+0xa/0x20
[<ffffffff802451a0>] ? kthread+0x0/0x80
[<ffffffff8020b660>] ? child_rip+0x0/0x20
Code: 18 4d 85 d2 74 25 41 ff cb 75 c4 4d 85 d2 74 16 8b 47 04 8d 4b 15 ba 01 

00 00 00 d3 e2 85 d0 75 05 09 d0 89 47 04 5b 4c 89 d0 c3 <0f> 0b eb fe 0f 0b eb fe 66 66 2e 0f 1f 84 00 00 00 00 00 41 57
RIP [<ffffffff8045281d>] radix_tree_tag_set+0x9d/0xb0
RSP <ffff8800ff97bce0>

---[ end trace 48e7c392f10d019e ]---
Aug 10 17:23:35 ganges ------------[ cut here ]------------
Aug 10 17:23:35 ganges Kernel BUG at ffffffff8045281d [verbose debug info 
unavailable]
Aug 10 17:23:35 ganges invalid opcode: 0000 [#1] SMP
Aug 10 17:23:35 ganges last sysfs 
file: /sys/devices/xen/pci-0/pci0000:00/0000:00:00.0/host3/target3:0:6/3:0:6:0/rev
Aug 10 17:23:35 ganges CPU 3
Aug 10 17:23:35 ganges Modules linked in: nfsd
Aug 10 17:23:35 ganges Pid: 227, comm: kswapd0 Not tainted 2.6.29-xen-r4 #13
Aug 10 17:23:35 ganges RIP: e030:[<ffffffff8045281d>]  [<ffffffff8045281d>] 
radix_tree_tag_set+0x9d/0xb0
Aug 10 17:23:35 ganges RSP: e02b:ffff8800ff97bce0  EFLAGS: 00010246
Aug 10 17:23:35 ganges RAX: 000000000000002d RBX: 0000000000000000 RCX: 
ffff8800c13ceba8
Aug 10 17:23:35 ganges RDX: 0000000000000000 RSI: 000000000022df71 RDI: 
ffff8800fe866e40
Aug 10 17:23:35 ganges RBP: ffff8800fe866e00 R08: 000000000000002d R09: 
000000000000000c
Aug 10 17:23:35 ganges R10: 0000000000000000 R11: 0000000000000003 R12: 
ffff8800fec59c00
Aug 10 17:23:35 ganges R13: ffff8800067f7938 R14: 0000000000000080 R15: 
000000000004828f
Aug 10 17:23:35 ganges FS:  00007f0617e8e6f0(0000) GS:ffff8800010061c0(0000) 
knlGS:0000000000000000
Aug 10 17:23:35 ganges CS:  e033 DS: 0000 ES: 0000 CR0: 000000008005003b
Aug 10 17:23:35 ganges CR2: 00007f187e14a110 CR3: 00000000fdcbd000 CR4: 
0000000000002620
Aug 10 17:23:35 ganges DR0: 0000000000000000 DR1: 0000000000000000 DR2: 
0000000000000000
Aug 10 17:23:35 ganges DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 
0000000000000400
Aug 10 17:23:35 ganges Process kswapd0 (pid: 227, threadinfo ffff8800ff97a000, 
task ffff8800ffb50540)
Aug 10 17:23:35 ganges Stack:
Aug 10 17:23:35 ganges ffff8800067f7840 ffffffff80422100 000000000004828f 
ffff8800067f7840
Aug 10 17:23:35 ganges ffff8800067f79c0 ffff8800ff97bd80 0000000000000012 
ffffffff8041229b
Aug 10 17:23:35 ganges ffff8800067f79c0 ffff8800067f79c0 ffff8800067f79c0 
ffffffff80421146
Aug 10 17:23:35 ganges Call Trace:
Aug 10 17:23:35 ganges [<ffffffff80422100>] ? 
xfs_inode_set_reclaim_tag+0x80/0xd0
Aug 10 17:23:35 ganges [<ffffffff8041229b>] ? xfs_reclaim+0x6b/0xe0
Aug 10 17:23:35 ganges [<ffffffff80421146>] ? xfs_fs_destroy_inode+0x36/0x60
Aug 10 17:23:35 ganges [<ffffffff802a8c12>] ? dispose_list+0xa2/0x150
Aug 10 17:23:35 ganges [<ffffffff802a8e9f>] ? shrink_icache_memory+0x1df/0x310
Aug 10 17:23:35 ganges [<ffffffff8026da44>] ? shrink_slab+0x124/0x180
Aug 10 17:23:35 ganges [<ffffffff8026e24d>] ? kswapd+0x3bd/0x600
Aug 10 17:23:35 ganges [<ffffffff8026b700>] ? isolate_pages_global+0x0/0x280
Aug 10 17:23:35 ganges [<ffffffff80245670>] ? 
autoremove_wake_function+0x0/0x30
Aug 10 17:23:35 ganges [<ffffffff802242fb>] ? __wake_up_common+0x5b/0x90
Aug 10 17:23:35 ganges [<ffffffff8026de90>] ? kswapd+0x0/0x600
Aug 10 17:23:35 ganges [<ffffffff802451eb>] ? kthread+0x4b/0x80
Aug 10 17:23:35 ganges [<ffffffff8020b66a>] ? child_rip+0xa/0x20
Aug 10 17:23:35 ganges [<ffffffff802451a0>] ? kthread+0x0/0x80
Aug 10 17:23:35 ganges [<ffffffff8020b660>] ? child_rip+0x0/0x20
Aug 10 17:23:35 ganges Code: 18 4d 85 d2 74 25 41 ff cb 75 c4 4d 85 d2 74 16 
8b 47 04 8d 4b 15 ba 01 00 00 00 d3 e2 85 d0 75 05 09 d0 89 47 04 5b 4c 89 d0 
c3 <0f> 0b eb fe 0f 0b eb fe 66 66 2e 0f 1f 84 00 00 00 00 00 41 57
Aug 10 17:23:35 ganges RIP  [<ffffffff8045281d>] radix_tree_tag_set+0x9d/0xb0
Aug 10 17:23:35 ganges RSP <ffff8800ff97bce0>
Aug 10 17:23:35 ganges ---[ end trace 48e7c392f10d019e ]---

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

I found that:
http://oss.sgi.com/archives/xfs-masters/2009-06/msg00056.html

------------------------------------------------------------
EasternGraphics - visualize your business

Christian Fischer
Administration
http://www.EasternGraphics.com
phone: +49 3677 678265

EasternGraphics GmbH - Albert-Einstein-Strasse 1 - DE-98693 Ilmenau
Geschaeftsfuehrer - Ekkehard Beier, Volker Blankenberg, Frank Wicht
Amtsgericht Jena - HRB 304052

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

It's a known bug that we just fixed in 2.6.31-rc. Can you check if this patch

	http://bugzilla.kernel.org/attachment.cgi?id=22590

applies to 2.6.29 (I produced it against 2.6.30)

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

Yes, I've seen this after sending my first mail.

If there are no patches against 2.6.29 then i have no other chance than trying this, or i switch to ext4. But what happens then?

What a bullshit, from one bug to the next, my company kills me.

Thanks for all comments and help.
Christian

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

backported to 2.6.29-xen-r4 from
http://code.google.com/p/gentoo-xen-kernel/downloads/list

Can you please review it?

The kernel complies and runs, but I can't test it.
I can't reproduce a crash as you could with 2.6.30 and 2.6.31. Nothing triggers it.

http://bugzilla.kernel.org/show_bug.cgi?id=13375

Christian

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

I'll reviewed it and do some QA over the weekend, and we'll send it to -stable then.

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

Did the patch fix the problem?

Andy

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

Hi Andrew,

nice to hear you.

All i can say is that the kernel compiles and runs.

On my testbox I can't reproduce the fs crash, seems 2.6.29.4 is more stable than 2.6.30/31.

I've tried Erics steps to reproduce for 2.6.30/31, tried stressing the nfs mount with iozone, tried recursive unpacking and removing eclipse (a lot of small files) and this in a lot of parallel tasks.

Nothing triggers the bug.
You know, if I can't reproduce it I can't say whether it is fixed or not.

I'll update one server this week with the patched kernel and switch nfs on again, and wait then for the next crash. Hope that comes never.

Christian

------------------------------------------------------------
EasternGraphics - visualize your business

Christian Fischer
Administration
http://www.EasternGraphics.com
phone: +49 3677 678265

EasternGraphics GmbH - Albert-Einstein-Strasse 1 - DE-98693 Ilmenau
Geschaeftsfuehrer - Ekkehard Beier, Volker Blankenberg, Frank Wicht
Amtsgericht Jena - HRB 304052

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

Well, attached the patch.

diff -Nurp linux-2.6.29-xen-r4.orig/fs/exportfs/Makefile linux-2.6.29-xen-r4/fs/exportfs/Makefile
@@ -1,6 +1,8 @@
 #
 # Makefile for the filesystem export support routines.
 
+EXTRA_CFLAGS += -I$(src)/../../fs/xfs -I$(src)/../../fs/xfs/linux-2.6
+
 obj-$(CONFIG_EXPORTFS) += exportfs.o
 
 exportfs-objs := expfs.o

diff -Nurp linux-2.6.29-xen-r4.orig/fs/exportfs/expfs.c linux-2.6.29-xen-r4/fs/exportfs/expfs.c
@@ -15,6 +15,11 @@
 #include <linux/mount.h>
 #include <linux/namei.h>
 #include <linux/sched.h>
+#include "xfs.h"
+#include "xfs_inum.h"
+#include "xfs_bmap_btree.h"
+#include "xfs_dinode.h"
+#include "xfs_inode.h"
 
 #define dprintk(fmt, args...) do{}while(0)
 

@@ -187,8 +192,25 @@ reconnect_path(struct vfsmount *mnt, str
 			 */
 			if (npd == pd)
 				noprogress = 0;
-			else
+			else {
 				printk("%s: npd != pd\n", __func__);
+				if (npd->d_inode) {
+					printk("npd = 0x%p, inode = 0%p, ino = 0x%llx\n",
+						npd, npd->d_inode,
+						(unsigned long long)npd->d_inode->i_ino);
+					printk("i_state = 0x%lx, i_flags = 0x%x\n",
+						npd->d_inode->i_state,
+						XFS_I(npd->d_inode)->i_flags);
+				}
+				if (pd->d_inode) {
+					printk("pd = 0x%p, inode = 0%p, ino = 0x%llx\n",
+						pd, pd->d_inode,
+						(unsigned long long)pd->d_inode->i_ino);
+					printk("i_state = 0x%lx, i_flags = 0x%x\n",
+						pd->d_inode->i_state,
+						XFS_I(pd->d_inode)->i_flags);
+				}
+			}
 			dput(npd);
 			dput(ppd);
 			if (IS_ROOT(pd)) {

diff -Nurp linux-2.6.29-xen-r4.orig/fs/inode.c linux-2.6.29-xen-r4/fs/inode.c
@@ -117,7 +117,7 @@ static void wake_up_inode(struct inode *
  * These are initializations that need to be done on every inode
  * allocation as the fields are not initialised by slab allocation.
  */
-struct inode *inode_init_always(struct super_block *sb, struct inode *inode)
+int inode_init_always(struct super_block *sb, struct inode *inode)
 {
 	static const struct address_space_operations empty_aops;
 	static struct inode_operations empty_iops;

@@ -147,14 +147,10 @@ struct inode *inode_init_always(struct s
 	inode->i_cdev = NULL;
 	inode->i_rdev = 0;
 	inode->dirtied_when = 0;
-	if (security_inode_alloc(inode)) {
-		if (inode->i_sb->s_op->destroy_inode)
-			inode->i_sb->s_op->destroy_inode(inode);
-		else
-			kmem_cache_free(inode_cachep, (inode));
-		return NULL;
-	}
-
+	
+	if (security_inode_alloc(inode))
+		return -ENOMEM;
+	
 	spin_lock_init(&inode->i_lock);
 	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
 

@@ -188,7 +184,7 @@ struct inode *inode_init_always(struct s
 	inode->i_private = NULL;
 	inode->i_mapping = mapping;
 
-	return inode;
+	return 0;
 }
 EXPORT_SYMBOL(inode_init_always);
 

@@ -201,22 +197,34 @@ static struct inode *alloc_inode(struct 
 	else
 		inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
 
-	if (inode)
-		return inode_init_always(sb, inode);
-	return NULL;
+	if (!inode)
+		return NULL;
+	
+	if (unlikely(inode_init_always(sb, inode))) {
+		if (inode->i_sb->s_op->destroy_inode)
+			inode->i_sb->s_op->destroy_inode(inode);
+		else
+			kmem_cache_free(inode_cachep, inode);
+	}
+	
+	return inode;
 }
 
-void destroy_inode(struct inode *inode) 
+void __destroy_inode(struct inode *inode)
 {
 	BUG_ON(inode_has_buffers(inode));
 	security_inode_free(inode);
+}
+EXPORT_SYMBOL(__destroy_inode);
+
+void destroy_inode(struct inode *inode)
+{
+	__destroy_inode(inode);
 	if (inode->i_sb->s_op->destroy_inode)
 		inode->i_sb->s_op->destroy_inode(inode);
 	else
 		kmem_cache_free(inode_cachep, (inode));
 }
-EXPORT_SYMBOL(destroy_inode);
-
 
 /*
  * These are initializations that only need to be done

diff -Nurp linux-2.6.29-xen-r4.orig/fs/xfs/linux-2.6/xfs_sync.c linux-2.6.29-xen-r4/fs/xfs/linux-2.6/xfs_sync.c
@@ -619,6 +619,17 @@ xfs_reclaim_inode(
 	return 0;
 }
 
+void
+__xfs_inode_set_reclaim_tag(
+	struct xfs_perag	*pag,
+	struct xfs_inode	*ip)
+{
+	radix_tree_tag_set(&pag->pag_ici_root,
+		XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino),
+		XFS_ICI_RECLAIM_TAG);
+	ip->i_flags |= XFS_IRECLAIMABLE;
+}
+
 /*
  * We set the inode flag atomically with the radix tree tag.
  * Once we get tag lookups on the radix tree, this inode flag

@@ -633,9 +644,7 @@ xfs_inode_set_reclaim_tag(
 
 	read_lock(&pag->pag_ici_lock);
 	spin_lock(&ip->i_flags_lock);
-	radix_tree_tag_set(&pag->pag_ici_root,
-			XFS_INO_TO_AGINO(mp, ip->i_ino), XFS_ICI_RECLAIM_TAG);
-	__xfs_iflags_set(ip, XFS_IRECLAIMABLE);
+	__xfs_inode_set_reclaim_tag(pag, ip);
 	spin_unlock(&ip->i_flags_lock);
 	read_unlock(&pag->pag_ici_lock);
 	xfs_put_perag(mp, pag);

@@ -643,27 +652,13 @@ xfs_inode_set_reclaim_tag(
 
 void
 __xfs_inode_clear_reclaim_tag(
-	xfs_mount_t	*mp,
-	xfs_perag_t	*pag,
-	xfs_inode_t	*ip)
+	struct xfs_perag	*pag,
+	struct xfs_inode	*ip)
 {
+	ip->i_flags &= ~XFS_IRECLAIMABLE;
 	radix_tree_tag_clear(&pag->pag_ici_root,
-			XFS_INO_TO_AGINO(mp, ip->i_ino), XFS_ICI_RECLAIM_TAG);
-}
-
-void
-xfs_inode_clear_reclaim_tag(
-	xfs_inode_t	*ip)
-{
-	xfs_mount_t	*mp = ip->i_mount;
-	xfs_perag_t	*pag = xfs_get_perag(mp, ip->i_ino);
-
-	read_lock(&pag->pag_ici_lock);
-	spin_lock(&ip->i_flags_lock);
-	__xfs_inode_clear_reclaim_tag(mp, pag, ip);
-	spin_unlock(&ip->i_flags_lock);
-	read_unlock(&pag->pag_ici_lock);
-	xfs_put_perag(mp, pag);
+		XFS_INO_TO_AGINO(ip->i_mount, ip->i_ino),
+		XFS_ICI_RECLAIM_TAG);
 }
 
 

diff -Nurp linux-2.6.29-xen-r4.orig/fs/xfs/linux-2.6/xfs_sync.h linux-2.6.29-xen-r4/fs/xfs/linux-2.6/xfs_sync.h
@@ -49,7 +49,6 @@ int xfs_reclaim_inode(struct xfs_inode *
 int xfs_reclaim_inodes(struct xfs_mount *mp, int noblock, int mode);
 
 void xfs_inode_set_reclaim_tag(struct xfs_inode *ip);
-void xfs_inode_clear_reclaim_tag(struct xfs_inode *ip);
-void __xfs_inode_clear_reclaim_tag(struct xfs_mount *mp, struct xfs_perag *pag,
-				struct xfs_inode *ip);
+void __xfs_inode_set_reclaim_tag(struct xfs_perag *pag, struct xfs_inode *ip);
+void __xfs_inode_clear_reclaim_tag(struct xfs_perag *pag, struct xfs_inode *ip);
 #endif

diff -Nurp linux-2.6.29-xen-r4.orig/fs/xfs/xfs_inode.h linux-2.6.29-xen-r4/fs/xfs/xfs_inode.h
@@ -309,23 +309,6 @@ static inline struct inode *VFS_I(struct
 }
 
 /*
- * Get rid of a partially initialized inode.
- *
- * We have to go through destroy_inode to make sure allocations
- * from init_inode_always like the security data are undone.
- *
- * We mark the inode bad so that it takes the short cut in
- * the reclaim path instead of going through the flush path
- * which doesn't make sense for an inode that has never seen the
- * light of day.
- */
-static inline void xfs_destroy_inode(struct xfs_inode *ip)
-{
-	make_bad_inode(VFS_I(ip));
-	return destroy_inode(VFS_I(ip));
-}
-
-/*
  * i_flags helper functions
  */
 static inline void

diff -Nurp linux-2.6.29-xen-r4.orig/include/linux/fs.h linux-2.6.29-xen-r4/include/linux/fs.h
@@ -1905,7 +1905,7 @@ extern loff_t default_llseek(struct file
 
 extern loff_t vfs_llseek(struct file *file, loff_t offset, int origin);
 
-extern struct inode * inode_init_always(struct super_block *, struct inode *);
+extern int inode_init_always(struct super_block *, struct inode *);
 extern void inode_init_once(struct inode *);
 extern void inode_add_to_lists(struct super_block *, struct inode *);
 extern void iput(struct inode *);

@@ -1932,6 +1932,7 @@ extern void __iget(struct inode * inode)
 extern void iget_failed(struct inode *);
 extern void clear_inode(struct inode *);
 extern void destroy_inode(struct inode *);
+extern void __destroy_inode(struct inode *);
 extern struct inode *new_inode(struct super_block *);
 extern int should_remove_suid(struct dentry *);
 extern int file_remove_suid(struct file *);

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs