Skip to contentSkip to conversation

nfsd stuckage

DISCUSSION Andrew Morton 5 participants 12 messages Jan 6, 2009 – Jan 8, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

I just built current mainline plus the just-sent 266 -mm patches.

The machine failed to power off when hit with `halt -pfn'. dmesg output:

[   37.087037] calling  rfcomm_init+0x0/0xb6 [rfcomm] @ 3946
[   37.087294] initcall rfcomm_init+0x0/0xb6 [rfcomm] returned 0 after 72 usecs
[   37.505855] calling  hidp_init+0x0/0x5e [hidp] @ 4046
[   37.506072] initcall hidp_init+0x0/0x5e [hidp] returned 0 after 28 usecs
[   37.636638] calling  init_autofs4_fs+0x0/0x23 [autofs4] @ 4081
[   37.636990] initcall init_autofs4_fs+0x0/0x23 [autofs4] returned 0 after 54 usecs
[   39.630075] calling  init_nlm+0x0/0x22 [lockd] @ 4264
[   39.630321] initcall init_nlm+0x0/0x22 [lockd] returned 0 after 59 usecs
[   39.690077] calling  init_rpcsec_gss+0x0/0x4a [auth_rpcgss] @ 4264
[   39.690281] initcall init_rpcsec_gss+0x0/0x4a [auth_rpcgss] returned 0 after 12 usecs
[   39.834034] calling  init_nfsd+0x0/0xe2 [nfsd] @ 4302
[   39.834471] initcall init_nfsd+0x0/0xe2 [nfsd] returned 0 after 236 usecs
[   39.924213] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[  672.162677] INFO: task nfsd4:4324 blocked for more than 480 seconds.
[  672.162706] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  672.162725]  ffff880251df1d60 0000000000000046 ffff88025e1c0580 ffff8802488013d8
[  672.162753]  ffff880251df1d20 ffff88024c49a7a0 ffff88025e088760 ffff88024c49ab18
[  672.162834]  000000002807ee00 00000000ffff59ec ffff880251df1d50 0000000000000282
[  672.162865] Call Trace:
[  672.162880]  [<ffffffff8052a1d8>] __mutex_lock_slowpath+0x6a/0xac
[  672.162895]  [<ffffffff8052a0c5>] mutex_lock+0x2c/0x30
[  672.162908]  [<ffffffff802c4747>] vfs_fsync+0x63/0xa9
[  672.162933]  [<ffffffffa048c74c>] nfsd_sync_dir+0x10/0x12 [nfsd]
[  672.162960]  [<ffffffffa04a5427>] nfsd4_sync_rec_dir+0x27/0x40 [nfsd]
[  672.162984]  [<ffffffffa04a592a>] nfsd4_recdir_purge_old+0x3d/0x6a [nfsd]
[  672.163023]  [<ffffffffa04a1745>] laundromat_main+0x62/0x225 [nfsd]
[  672.163049]  [<ffffffffa04a16e3>] ? laundromat_main+0x0/0x225 [nfsd]
[  672.163064]  [<ffffffff8024b4a7>] run_workqueue+0x8d/0x124
[  672.163076]  [<ffffffff8024b5e0>] ? worker_thread+0x0/0xe5
[  672.163089]  [<ffffffff8024b6b8>] worker_thread+0xd8/0xe5
[  672.163102]  [<ffffffff8024e8cc>] ? autoremove_wake_function+0x0/0x36
[  672.163115]  [<ffffffff8024b5e0>] ? worker_thread+0x0/0xe5
[  672.163127]  [<ffffffff8024e5e0>] kthread+0x44/0x6b
[  672.163140]  [<ffffffff8020cfba>] child_rip+0xa/0x20
[  672.163151]  [<ffffffff8024e59c>] ? kthread+0x0/0x6b
[  672.163162]  [<ffffffff8020cfb0>] ? child_rip+0x0/0x20
[ 1204.739381] INFO: task nfsd4:4324 blocked for more than 480 seconds.
[ 1204.739415] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 1204.739436]  ffff880251df1d60 0000000000000046 ffff88025e1c0580 ffff8802488013d8
[ 1204.739523]  ffff880251df1d20 ffff88024c49a7a0 ffff88025e088760 ffff88024c49ab18
[ 1204.739551]  000000002807ee00 00000000ffff59ec ffff880251df1d50 0000000000000282
[ 1204.739579] Call Trace:

This didn't happen in linux-next a week or so ago.

#1 lore raw
Quoted 4 lines · Andrew Morton

I just built current mainline plus the just-sent 266 -mm patches.

The machine failed to power off when hit with `halt -pfn'. dmesg output:

Christoph, can you live with this for now?

--b.

commit 33e3950dc2eae7484e79685083c304d93013e3ec

Author: J. Bruce Fields <[email protected]>
Date:   Tue Jan 6 13:37:03 2009 -0500

nfsd: fix double-locks of directory mutex

A number of nfsd operations depend on the i_mutex to cover more code than just the fsync, so the approach of 4c728ef583b3d8 "add a vfs_fsync helper" doesn't work for nfsd. Revert the parts of those patches that touch nfsd, and remove the logic from vfs_nfsd that was needed only for the special case of nfsd.

fs/nfsd/vfs.c +31 -3
index 44aa92a..6e50aaa 100644
@@ -744,16 +744,44 @@ nfsd_close(struct file *filp)
 	fput(filp);
 }
 
+/*
+ * Sync a file
+ * As this calls fsync (not fdatasync) there is no need for a write_inode
+ * after it.
+ */
+static inline int nfsd_dosync(struct file *filp, struct dentry *dp,
+			      const struct file_operations *fop)
+{
+	struct inode *inode = dp->d_inode;
+	int (*fsync) (struct file *, struct dentry *, int);
+	int err;
+
+	err = filemap_fdatawrite(inode->i_mapping);
+	if (err == 0 && fop && (fsync = fop->fsync))
+		err = fsync(filp, dp, 0);
+	if (err == 0)
+		err = filemap_fdatawait(inode->i_mapping);
+
+	return err;
+}
+
 static int
 nfsd_sync(struct file *filp)
 {
-	return vfs_fsync(filp, filp->f_path.dentry, 0);
+        int err;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	dprintk("nfsd: sync file %s\n", filp->f_path.dentry->d_name.name);
+	mutex_lock(&inode->i_mutex);
+	err=nfsd_dosync(filp, filp->f_path.dentry, filp->f_op);
+	mutex_unlock(&inode->i_mutex);
+
+	return err;
 }
 
 int
-nfsd_sync_dir(struct dentry *dentry)
+nfsd_sync_dir(struct dentry *dp)
 {
-	return vfs_fsync(NULL, dentry, 0);
+	return nfsd_dosync(NULL, dp, dp->d_inode->i_fop);
 }
 
 /*
fs/sync.c +2 -16
index 0921d6d..8e0a656 100644
@@ -83,10 +83,6 @@ int file_fsync(struct file *filp, struct dentry *dentry, int datasync)
  *
  * Write back data and metadata for @file to disk.  If @datasync is
  * set only metadata needed to access modified file data is written.
- *
- * In case this function is called from nfsd @file may be %NULL and
- * only @dentry is set.  This can only happen when the filesystem
- * implements the export_operations API.
  */
 int vfs_fsync(struct file *file, struct dentry *dentry, int datasync)
 {
@@ -94,18 +90,8 @@ int vfs_fsync(struct file *file, struct dentry *dentry, int datasync)
 	struct address_space *mapping;
 	int err, ret;
 
-	/*
-	 * Get mapping and operations from the file in case we have
-	 * as file, or get the default values for them in case we
-	 * don't have a struct file available.  Damn nfsd..
-	 */
-	if (file) {
-		mapping = file->f_mapping;
-		fop = file->f_op;
-	} else {
-		mapping = dentry->d_inode->i_mapping;
-		fop = dentry->d_inode->i_fop;
-	}
+	mapping = file->f_mapping;
+	fop = file->f_op;
 
 	if (!fop || !fop->fsync) {
 		ret = -EINVAL;
#2 lore raw
Quoted 7 lines · J. Bruce Fields
Quoted 4 lines · Andrew Morton

I just built current mainline plus the just-sent 266 -mm patches.

The machine failed to power off when hit with `halt -pfn'. dmesg output:

Christoph, can you live with this for now?

nfsd part is well, livable. But the fs/sync.c is buggy as stackable filesystems can call vfs_fsync with a NULL pointer due to nfs calling it that way, so please drop that hunk.

#3 lore raw
Quoted 5 lines · Christoph Hellwig
Quoted 4 lines · J. Bruce Fields
Quoted 3 lines · Andrew Morton

I just built current mainline plus the just-sent 266 -mm patches.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

The machine failed to power off when hit with `halt -pfn'. dmesg output:

Christoph, can you live with this for now?

nfsd part is well, livable. But the fs/sync.c is buggy as stackable filesystems can call vfs_fsync with a NULL pointer due to nfs calling it that way, so please drop that hunk.

Whoops, OK, I didn't understand that. I'll drop that hunk, retest, then submit--should take a few minutes.

--b.

#4 lore raw
Quoted 8 lines · bfields
Quoted 7 lines · Christoph Hellwig
Quoted 6 lines · J. Bruce Fields
Quoted 4 lines · Andrew Morton

I just built current mainline plus the just-sent 266 -mm patches.

The machine failed to power off when hit with `halt -pfn'. dmesg output:

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

Christoph, can you live with this for now?

nfsd part is well, livable. But the fs/sync.c is buggy as stackable filesystems can call vfs_fsync with a NULL pointer due to nfs calling it that way, so please drop that hunk.

Whoops, OK, I didn't understand that. I'll drop that hunk, retest, then submit--should take a few minutes.

So this works for me--and I guess I may as well submit it in a pull request. But: it sounds like we still have a regression for ecryptfs? (Since on ecryptfs export nfsd will still try to get the mutex twice on create, unlink, etc.) Maybe we should just revert 4c728ef583b3d82266584da5cb068294c09df31e entirely for now?

--b.

commit 3fbc5c762bd9f9ff52fe7b5b09398a3cff0e8415

Author: J. Bruce Fields <[email protected]>
Date:   Tue Jan 6 13:37:03 2009 -0500

nfsd: fix double-locks of directory mutex

A number of nfsd operations depend on the i_mutex to cover more code than just the fsync, so the approach of 4c728ef583b3d8 "add a vfs_fsync helper" doesn't work for nfsd. Revert the parts of those patches that touch nfsd.

Note: we can't, however, remove the logic from vfs_fsync that was needed only for the special case of nfsd, because a vfs_fsync(NULL,...) call can still result indirectly from a stackable filesystem that was called by nfsd.

fs/nfsd/vfs.c +31 -3
index 44aa92a..6e50aaa 100644
@@ -744,16 +744,44 @@ nfsd_close(struct file *filp)
 	fput(filp);
 }
 
+/*
+ * Sync a file
+ * As this calls fsync (not fdatasync) there is no need for a write_inode
+ * after it.
+ */
+static inline int nfsd_dosync(struct file *filp, struct dentry *dp,
+			      const struct file_operations *fop)
+{
+	struct inode *inode = dp->d_inode;
+	int (*fsync) (struct file *, struct dentry *, int);
+	int err;
+
+	err = filemap_fdatawrite(inode->i_mapping);
+	if (err == 0 && fop && (fsync = fop->fsync))
+		err = fsync(filp, dp, 0);
+	if (err == 0)
+		err = filemap_fdatawait(inode->i_mapping);
+
+	return err;
+}
+
 static int
 nfsd_sync(struct file *filp)
 {
-	return vfs_fsync(filp, filp->f_path.dentry, 0);
+        int err;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	dprintk("nfsd: sync file %s\n", filp->f_path.dentry->d_name.name);
+	mutex_lock(&inode->i_mutex);
+	err=nfsd_dosync(filp, filp->f_path.dentry, filp->f_op);
+	mutex_unlock(&inode->i_mutex);
+
+	return err;
 }
 
 int
-nfsd_sync_dir(struct dentry *dentry)
+nfsd_sync_dir(struct dentry *dp)
 {
-	return vfs_fsync(NULL, dentry, 0);
+	return nfsd_dosync(NULL, dp, dp->d_inode->i_fop);
 }
 
 /*
#5 lore raw
Quoted 1 line · On Tue, 6 Jan 2009 19:15:01 -0500 "J. Bruce Fields" <[email protected]>…

nfsd: fix double-locks of directory mutex

grumble.

Quoted 11 lines
fs/nfsd/vfs.c +10 -0
+/*
+ * Sync a file
+ * As this calls fsync (not fdatasync) there is no need for a write_inode
+ * after it.
+ */
+static inline int nfsd_dosync(struct file *filp, struct dentry *dp,
+			      const struct file_operations *fop)
+{
+	struct inode *inode = dp->d_inode;
+	int (*fsync) (struct file *, struct dentry *, int);

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 766

Quoted 10 lines
fs/nfsd/vfs.c +10 -0
+	int err;
+
+	err = filemap_fdatawrite(inode->i_mapping);
+	if (err == 0 && fop && (fsync = fop->fsync))
+		err = fsync(filp, dp, 0);
+	if (err == 0)
+		err = filemap_fdatawait(inode->i_mapping);
+
+	return err;
+}

This function is HUGE! And hardly a fastpath.

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 775

Quoted 9 lines

static int
nfsd_sync(struct file *filp)
{

fs/nfsd/vfs.c +5 -1
-	return vfs_fsync(filp, filp->f_path.dentry, 0);
+        int err;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	dprintk("nfsd: sync file %s\n", filp->f_path.dentry->d_name.name);
+	mutex_lock(&inode->i_mutex);
+	err=nfsd_dosync(filp, filp->f_path.dentry, filp->f_op);

(checkpatch?)

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 785

Quoted 12 lines
fs/nfsd/vfs.c +5 -2
+	mutex_unlock(&inode->i_mutex);
+
+	return err;
 }
 
 int
-nfsd_sync_dir(struct dentry *dentry)
+nfsd_sync_dir(struct dentry *dp)
 {
-	return vfs_fsync(NULL, dentry, 0);
+	return nfsd_dosync(NULL, dp, dp->d_inode->i_fop);
 }

And we expand it twice.

#6 lore raw
Quoted 6 lines · Andrew Morton
Quoted 1 line · On Tue, 6 Jan 2009 19:15:01 -0500 "J. Bruce Fields" <[email protected]>…

nfsd: fix double-locks of directory mutex

grumble.

This is literally just a revert of part of 4c728ef583b3d822; if you'd like me to clean up this stuff while I'm there, I'm happy to.

--b.

Quoted 49 lines
Quoted 10 lines
fs/nfsd/vfs.c +10 -0
+/*
+ * Sync a file
+ * As this calls fsync (not fdatasync) there is no need for a write_inode
+ * after it.
+ */
+static inline int nfsd_dosync(struct file *filp, struct dentry *dp,
+			      const struct file_operations *fop)
+{
+	struct inode *inode = dp->d_inode;
+	int (*fsync) (struct file *, struct dentry *, int);

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 766

Quoted 10 lines
fs/nfsd/vfs.c +10 -0
+	int err;
+
+	err = filemap_fdatawrite(inode->i_mapping);
+	if (err == 0 && fop && (fsync = fop->fsync))
+		err = fsync(filp, dp, 0);
+	if (err == 0)
+		err = filemap_fdatawait(inode->i_mapping);
+
+	return err;
+}

This function is HUGE! And hardly a fastpath.

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 775

Quoted 9 lines

static int
nfsd_sync(struct file *filp)
{

fs/nfsd/vfs.c +5 -1
-	return vfs_fsync(filp, filp->f_path.dentry, 0);
+        int err;
+	struct inode *inode = filp->f_path.dentry->d_inode;
+	dprintk("nfsd: sync file %s\n", filp->f_path.dentry->d_name.name);
+	mutex_lock(&inode->i_mutex);
+	err=nfsd_dosync(filp, filp->f_path.dentry, filp->f_op);

(checkpatch?)

fs/nfsd/vfs.c nfsd_close(struct file *filp) · line 785

Quoted 12 lines
fs/nfsd/vfs.c +5 -2
+	mutex_unlock(&inode->i_mutex);
+
+	return err;
 }
 
 int
-nfsd_sync_dir(struct dentry *dentry)
+nfsd_sync_dir(struct dentry *dp)
 {
-	return vfs_fsync(NULL, dentry, 0);
+	return nfsd_dosync(NULL, dp, dp->d_inode->i_fop);
 }

And we expand it twice.

#7 lore raw
Quoted 10 lines · J. Bruce Fields
Quoted 6 lines · Andrew Morton
Quoted 1 line · On Tue, 6 Jan 2009 19:15:01 -0500 "J. Bruce Fields" <[email protected]>…

nfsd: fix double-locks of directory mutex

grumble.

This is literally just a revert of part of 4c728ef583b3d822; if you'd like me to clean up this stuff while I'm there, I'm happy to.

Please leave it as the revert. NFSD really needs to use vfs_fsync eventually so we can sort out our ->fsync usage. I suspect the best way to get there is to to the i_mutex removal for fsync earlier than planned, but I'll need to audit the filesystems first.

#8 lore raw
Quoted 7 lines · Christoph Hellwig
Quoted 6 lines · J. Bruce Fields
Quoted 5 lines · Andrew Morton
Quoted 1 line · On Tue, 6 Jan 2009 19:15:01 -0500 "J. Bruce Fields" <[email protected]>…

nfsd: fix double-locks of directory mutex

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

grumble.

This is literally just a revert of part of 4c728ef583b3d822; if you'd like me to clean up this stuff while I'm there, I'm happy to.

Please leave it as the revert. NFSD really needs to use vfs_fsync eventually so we can sort out our ->fsync usage.

OK. Mind if we just revert the whole commit for now? With the double-lock regression is still there for ecryptfs exports, then I'd rather do a simple revert of the whole patch and not try to pick out just the fs/nfsd/vfs.c part.

--b.

Quoted 3 lines

I suspect the best way to get there is to to the i_mutex removal for fsync earlier than planned, but I'll need to audit the filesystems first.

#9 lore raw
Quoted 4 lines · J. Bruce Fields

OK. Mind if we just revert the whole commit for now? With the double-lock regression is still there for ecryptfs exports, then I'd rather do a simple revert of the whole patch and not try to pick out just the fs/nfsd/vfs.c part.

Umm, exporting ecryptfs would previously take the lower i_mutex in the ecryptfs fsync method and now does in vfs_fsync, there should be no changed in behaviour.

#10 lore raw
In reply to Andrew Morton
Quoted 19 lines · Andrew Morton

I just built current mainline plus the just-sent 266 -mm patches.

The machine failed to power off when hit with `halt -pfn'. dmesg output:

[  672.162677] INFO: task nfsd4:4324 blocked for more than 480 seconds.
[  672.162706] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[  672.162725]  ffff880251df1d60 0000000000000046 ffff88025e1c0580 ffff8802488013d8
[  672.162753]  ffff880251df1d20 ffff88024c49a7a0 ffff88025e088760 ffff88024c49ab18
[  672.162834]  000000002807ee00 00000000ffff59ec ffff880251df1d50 0000000000000282
[  672.162865] Call Trace:
[  672.162880]  [<ffffffff8052a1d8>] __mutex_lock_slowpath+0x6a/0xac
[  672.162895]  [<ffffffff8052a0c5>] mutex_lock+0x2c/0x30
[  672.162908]  [<ffffffff802c4747>] vfs_fsync+0x63/0xa9
[  672.162933]  [<ffffffffa048c74c>] nfsd_sync_dir+0x10/0x12 [nfsd]
[  672.162960]  [<ffffffffa04a5427>] nfsd4_sync_rec_dir+0x27/0x40 [nfsd]
[  672.162984]  [<ffffffffa04a592a>] nfsd4_recdir_purge_old+0x3d/0x6a [nfsd]
[  672.163023]  [<ffffffffa04a1745>] laundromat_main+0x62/0x225 [nfsd]
[  672.163049]  [<ffffffffa04a16e3>] ? laundromat_main+0x0/0x225 [nfsd]
[  672.163064]  [<ffffffff8024b4a7>] run_workqueue+0x8d/0x124
Quoted 8 lines
[  672.163076]  [<ffffffff8024b5e0>] ? worker_thread+0x0/0xe5
[  672.163089]  [<ffffffff8024b6b8>] worker_thread+0xd8/0xe5
[  672.163102]  [<ffffffff8024e8cc>] ? autoremove_wake_function+0x0/0x36
[  672.163115]  [<ffffffff8024b5e0>] ? worker_thread+0x0/0xe5
[  672.163127]  [<ffffffff8024e5e0>] kthread+0x44/0x6b
[  672.163140]  [<ffffffff8020cfba>] child_rip+0xa/0x20
[  672.163151]  [<ffffffff8024e59c>] ? kthread+0x0/0x6b
[  672.163162]  [<ffffffff8020cfb0>] ? child_rip+0x0/0x20

FWIW lockdep seems to warn about this...

All I have to do to trigger this is boot the machine and let it sit for a few minutes.

[  113.552497] =============================================
[  113.553289] [ INFO: possible recursive locking detected ]
[  113.553289] 2.6.28-tip #592                              
[  113.553289] ---------------------------------------------
[  113.553289] nfsd4/1914 is trying to acquire lock:        
[  113.553289]  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffff802e7e5e>] vfs_fsync+0x6c/0xb1
[  113.553289]                                                                                
[  113.553289] but task is already holding lock:                                              
[  113.553289]  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffffa0190727>] nfsd4_sync_rec_dir+0x22/0x47 [nfsd]                                                                                                        
[  113.553289]                                                                                             
[  113.553289] other info that might help us debug this:                                                   
[  113.553289] 4 locks held by nfsd4/1914:                                                                 
[  113.553289]  #0:  (nfsd4){--..}, at: [<ffffffff80252303>] run_workqueue+0xb6/0x21b                      
[  113.553289]  #1:  ((laundromat_work).work){--..}, at: [<ffffffff80252303>] run_workqueue+0xb6/0x21b     
[  113.553289]  #2:  (client_mutex){--..}, at: [<ffffffffa018bd05>] laundromat_main+0x33/0x24e [nfsd]      
[  113.553289]  #3:  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffffa0190727>] nfsd4_sync_rec_dir+0x22/0x47 [nfsd]                                                                                                   
[  113.553289]                                                                                             
[  113.553289] stack backtrace:                                                                            
[  113.553289] Pid: 1914, comm: nfsd4 Not tainted 2.6.28-tip #592                                          
[  113.553289] Call Trace:                                                                                 
[  113.553289]  [<ffffffff80266987>] __lock_acquire+0xe42/0x161a                                           
[  113.553289]  [<ffffffff80288857>] ? __call_rcu+0x7a/0x107                                               
[  113.553289]  [<ffffffff802671b4>] lock_acquire+0x55/0x71                                                
[  113.553289]  [<ffffffff802e7e5e>] ? vfs_fsync+0x6c/0xb1                                                 
[  113.553289]  [<ffffffff805568d0>] mutex_lock_nested+0x4e/0x320                                          
[  113.553289]  [<ffffffff802e7e5e>] ? vfs_fsync+0x6c/0xb1                                                 
[  113.553289]  [<ffffffff8029bde0>] ? __filemap_fdatawrite_range+0x57/0x5f                                
[  113.553289]  [<ffffffff802e7e5e>] vfs_fsync+0x6c/0xb1                                                   
[  113.553289]  [<ffffffffa0176f8f>] nfsd_sync_dir+0x15/0x17 [nfsd]                                        
[  113.553289]  [<ffffffffa0190733>] nfsd4_sync_rec_dir+0x2e/0x47 [nfsd]                                   
[  113.553289]  [<ffffffffa0190791>] nfsd4_recdir_purge_old+0x45/0x73 [nfsd]                               
[  113.553289]  [<ffffffffa018bd44>] laundromat_main+0x72/0x24e [nfsd]                                     
[  113.553289]  [<ffffffff80252355>] run_workqueue+0x108/0x21b                                             
[  113.553289]  [<ffffffff80252303>] ? run_workqueue+0xb6/0x21b                                            
[  113.553289]  [<ffffffffa018bcd2>] ? laundromat_main+0x0/0x24e [nfsd]                                    
[  113.553289]  [<ffffffff8025254d>] worker_thread+0xe5/0xf6                                               
[  113.553289]  [<ffffffff80256615>] ? autoremove_wake_function+0x0/0x3d                                   
[  113.553289]  [<ffffffff80252468>] ? worker_thread+0x0/0xf6                                              
[  113.553289]  [<ffffffff80256200>] kthread+0x4e/0x7b                                                     
[  113.553289]  [<ffffffff8020d51a>] child_rip+0xa/0x20                                                    
[  113.553289]  [<ffffffff8020cec0>] ? restore_args+0x0/0x30                                               
[  113.553289]  [<ffffffff802561b2>] ? kthread+0x0/0x7b                                                    
[  113.553289]  [<ffffffff8020d510>] ? child_rip+0x0/0x20 
#11 lore raw
Quoted 4 lines · Peter Zijlstra

FWIW lockdep seems to warn about this...

All I have to do to trigger this is boot the machine and let it sit for a few minutes.

Linus merged a fix (9a8d248e2d2 "nfsd: fix double-locks of directory mutex") last night. If you still see warnings after that, let us know.

--b.

Quoted 46 lines
[  113.552497] =============================================
[  113.553289] [ INFO: possible recursive locking detected ]
[  113.553289] 2.6.28-tip #592                              
[  113.553289] ---------------------------------------------
[  113.553289] nfsd4/1914 is trying to acquire lock:        
[  113.553289]  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffff802e7e5e>] vfs_fsync+0x6c/0xb1
[  113.553289]                                                                                
[  113.553289] but task is already holding lock:                                              
[  113.553289]  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffffa0190727>] nfsd4_sync_rec_dir+0x22/0x47 [nfsd]                                                                                                        
[  113.553289]                                                                                             
[  113.553289] other info that might help us debug this:                                                   
[  113.553289] 4 locks held by nfsd4/1914:                                                                 
[  113.553289]  #0:  (nfsd4){--..}, at: [<ffffffff80252303>] run_workqueue+0xb6/0x21b                      
[  113.553289]  #1:  ((laundromat_work).work){--..}, at: [<ffffffff80252303>] run_workqueue+0xb6/0x21b     
[  113.553289]  #2:  (client_mutex){--..}, at: [<ffffffffa018bd05>] laundromat_main+0x33/0x24e [nfsd]      
[  113.553289]  #3:  (&type->i_mutex_dir_key#4){--..}, at: [<ffffffffa0190727>] nfsd4_sync_rec_dir+0x22/0x47 [nfsd]                                                                                                   
[  113.553289]                                                                                             
[  113.553289] stack backtrace:                                                                            
[  113.553289] Pid: 1914, comm: nfsd4 Not tainted 2.6.28-tip #592                                          
[  113.553289] Call Trace:                                                                                 
[  113.553289]  [<ffffffff80266987>] __lock_acquire+0xe42/0x161a                                           
[  113.553289]  [<ffffffff80288857>] ? __call_rcu+0x7a/0x107                                               
[  113.553289]  [<ffffffff802671b4>] lock_acquire+0x55/0x71                                                
[  113.553289]  [<ffffffff802e7e5e>] ? vfs_fsync+0x6c/0xb1                                                 
[  113.553289]  [<ffffffff805568d0>] mutex_lock_nested+0x4e/0x320                                          
[  113.553289]  [<ffffffff802e7e5e>] ? vfs_fsync+0x6c/0xb1                                                 
[  113.553289]  [<ffffffff8029bde0>] ? __filemap_fdatawrite_range+0x57/0x5f                                
[  113.553289]  [<ffffffff802e7e5e>] vfs_fsync+0x6c/0xb1                                                   
[  113.553289]  [<ffffffffa0176f8f>] nfsd_sync_dir+0x15/0x17 [nfsd]                                        
[  113.553289]  [<ffffffffa0190733>] nfsd4_sync_rec_dir+0x2e/0x47 [nfsd]                                   
[  113.553289]  [<ffffffffa0190791>] nfsd4_recdir_purge_old+0x45/0x73 [nfsd]                               
[  113.553289]  [<ffffffffa018bd44>] laundromat_main+0x72/0x24e [nfsd]                                     
[  113.553289]  [<ffffffff80252355>] run_workqueue+0x108/0x21b                                             
[  113.553289]  [<ffffffff80252303>] ? run_workqueue+0xb6/0x21b                                            
[  113.553289]  [<ffffffffa018bcd2>] ? laundromat_main+0x0/0x24e [nfsd]                                    
[  113.553289]  [<ffffffff8025254d>] worker_thread+0xe5/0xf6                                               
[  113.553289]  [<ffffffff80256615>] ? autoremove_wake_function+0x0/0x3d                                   
[  113.553289]  [<ffffffff80252468>] ? worker_thread+0x0/0xf6                                              
[  113.553289]  [<ffffffff80256200>] kthread+0x4e/0x7b                                                     
[  113.553289]  [<ffffffff8020d51a>] child_rip+0xa/0x20                                                    
[  113.553289]  [<ffffffff8020cec0>] ? restore_args+0x0/0x30                                               
[  113.553289]  [<ffffffff802561b2>] ? kthread+0x0/0x7b                                                    
[  113.553289]  [<ffffffff8020d510>] ? child_rip+0x0/0x20