Skip to contentSkip to conversation

pnfs: devide put_lseg and return_layout_barrier into different workqueue

PATCH In review Zhang Jingwang 4 participants 9 messages May 17, 2010 – May 24, 2010 1 message not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 pnfs: devide put_lseg and return_layout_barrier into different workqueue

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

---
 fs/nfs/pnfs.c |   58 +++++++++++++++++++++++++++++++++++++++-----------------
 1 files changed, 40 insertions(+), 18 deletions(-)
fs/nfs/pnfs.c +40 -18
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
  * cleanup.
  */
 static void
+pnfs_write_retry(struct work_struct *work)
+{
+	struct rpc_task *task;
+	struct nfs_write_data *wdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	wdata = container_of(task, struct nfs_write_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = wdata->args.offset;
+	range.length = wdata->args.count;
+	_pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE);
+	pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+			    wdata->pdata.call_ops, wdata->pdata.how);
+}
+
+static void
 pnfs_writeback_done(struct nfs_write_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
 	}
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_RW,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-		pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-				    pdata->call_ops, pdata->how);
+		INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);
 	}
 }
 
@@ -1798,6 +1809,24 @@ out:
  * read_pagelist is done
  */
 static void
+pnfs_read_retry(struct work_struct *work)
+{
+	struct rpc_task *task;
+	struct nfs_read_data *rdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	rdata = container_of(task, struct nfs_read_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = rdata->args.offset;
+	range.length = rdata->args.count;
+	_pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE);
+	pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+			   rdata->pdata.call_ops);
+}
+
+static void
 pnfs_read_done(struct nfs_read_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)
 	dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_ANY,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-		pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-				   pdata->call_ops);
+		INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);
 	}
 }
 
Signature · 1 line
1.6.2.5
#1 lore raw
Quoted 4 lines · Zhang Jingwang

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

What is the stack trace of the deadlock?

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Or you can wait for the new state machine.

Boaz

Quoted 96 lines
---
 fs/nfs/pnfs.c |   58 +++++++++++++++++++++++++++++++++++++++-----------------
 1 files changed, 40 insertions(+), 18 deletions(-)
fs/nfs/pnfs.c +40 -18
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
  * cleanup.
  */
 static void
+pnfs_write_retry(struct work_struct *work)
+{
+	struct rpc_task *task;
+	struct nfs_write_data *wdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	wdata = container_of(task, struct nfs_write_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = wdata->args.offset;
+	range.length = wdata->args.count;
+	_pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE);
+	pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+			    wdata->pdata.call_ops, wdata->pdata.how);
+}
+
+static void
 pnfs_writeback_done(struct nfs_write_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
 	}
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_RW,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-		pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-				    pdata->call_ops, pdata->how);
+		INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);
 	}
 }
 
@@ -1798,6 +1809,24 @@ out:
  * read_pagelist is done
  */
 static void
+pnfs_read_retry(struct work_struct *work)
+{
+	struct rpc_task *task;
+	struct nfs_read_data *rdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	rdata = container_of(task, struct nfs_read_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = rdata->args.offset;
+	range.length = rdata->args.count;
+	_pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE);
+	pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+			   rdata->pdata.call_ops);
+}
+
+static void
 pnfs_read_done(struct nfs_read_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)
 	dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_ANY,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-		pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-				   pdata->call_ops);
+		INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);
 	}
 }
 
#2 lore raw
Quoted 18 lines · Boaz Harrosh
Quoted 4 lines · Zhang Jingwang

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

What is the stack trace of the deadlock?

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Quoted 4 lines

Or you can wait for the new state machine.

Boaz

BTW: I agree that current code is crap. Do to bugs in the osd library we never return -EAGAIN. so I never tried that code. But it should theoretically trigger when an OSD reboots or a network connection fails.

Quoted 3 lines
Quoted 3 lines

---

Thanks
Boaz

#3 lore raw
In reply to Boaz Harrosh

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 8 lines · Zhang Jingwang
Quoted 4 lines

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

Quoted 8 lines

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will not help.

Quoted 105 lines

Boaz

Quoted 86 lines

---
 fs/nfs/pnfs.c |   58 +++++++++++++++++++++++++++++++++++++++-----------------

 1 files changed, 40 insertions(+), 18 deletions(-)
fs/nfs/pnfs.c +0 -0
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
   * cleanup.
   */

 static void

fs/nfs/pnfs.c +18 -0
+pnfs_write_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_write_data *wdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     wdata = container_of(task, struct nfs_write_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = wdata->args.offset;
+     range.length = wdata->args.count;
+     _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+                         wdata->pdata.call_ops, wdata->pdata.how);
+}
+
+static void

 pnfs_writeback_done(struct nfs_write_data *data) {
      struct pnfs_call_data *pdata = &data->pdata;

fs/nfs/pnfs.c +2 -9
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
       }
 
       if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_RW,
-                     .offset = data->args.offset,
-                     .length = data->args.count,
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-                                 pdata->call_ops, pdata->how);
+             INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
       }
  }
 
@@ -1798,6 +1809,24 @@ out:
   * read_pagelist is done
   */

 static void

fs/nfs/pnfs.c +18 -0
+pnfs_read_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_read_data *rdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     rdata = container_of(task, struct nfs_read_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = rdata->args.offset;
+     range.length = rdata->args.count;
+     _pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+                        rdata->pdata.call_ops);
+}
+
+static void

 pnfs_read_done(struct nfs_read_data *data) {
      struct pnfs_call_data *pdata = &data->pdata;

fs/nfs/pnfs.c +0 -4
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)
       dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);
 
       if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_ANY,
-                     .offset = data->args.offset,
-                     .length = data->args.count,

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1841

Quoted 10 lines
fs/nfs/pnfs.c +2 -5
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-                                pdata->call_ops);
+             INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
       }
  }
 

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Signature · 5 lines
Zhang Jingwang
National Research Centre for High Performance Computers
Institute of Computing Technology, Chinese Academy of Sciences
No. 6, South Kexueyuan Road, Haidian District
Beijing, China
#4 lore raw
Quoted 23 lines · Zhang Jingwang

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 8 lines · Zhang Jingwang
Quoted 4 lines

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

Quoted 7 lines

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Quoted 4 lines
Quoted 1 line

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will not help.

I think what you did is right for the time being and I'll merge it until we have something better.
The state machine should help in this case since it will effectively switch contexts between two tasks rather than blocking synchronously.

Benny

Quoted 108 lines
Quoted 105 lines

Boaz

Quoted 86 lines
---
 fs/nfs/pnfs.c |   58 +++++++++++++++++++++++++++++++++++++++-----------------
 1 files changed, 40 insertions(+), 18 deletions(-)
fs/nfs/pnfs.c +38 -13
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
  * cleanup.
  */
 static void
+pnfs_write_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_write_data *wdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     wdata = container_of(task, struct nfs_write_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = wdata->args.offset;
+     range.length = wdata->args.count;
+     _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+                         wdata->pdata.call_ops, wdata->pdata.how);
+}
+
+static void
 pnfs_writeback_done(struct nfs_write_data *data)
 {
      struct pnfs_call_data *pdata = &data->pdata;
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
      }
 
      if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_RW,
-                     .offset = data->args.offset,
-                     .length = data->args.count,
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-                                 pdata->call_ops, pdata->how);
+             INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
      }
 }
 
@@ -1798,6 +1809,24 @@ out:
  * read_pagelist is done
  */
 static void
+pnfs_read_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_read_data *rdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     rdata = container_of(task, struct nfs_read_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = rdata->args.offset;
+     range.length = rdata->args.count;
+     _pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+                        rdata->pdata.call_ops);
+}
+
+static void
 pnfs_read_done(struct nfs_read_data *data)
 {
      struct pnfs_call_data *pdata = &data->pdata;
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)
      dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);
 
      if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_ANY,
-                     .offset = data->args.offset,
-                     .length = data->args.count,

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1841

Quoted 10 lines
fs/nfs/pnfs.c +2 -5
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-                                pdata->call_ops);
+             INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
      }
 }
 

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#5 lore raw
Quoted 29 lines · Benny Halevy
Quoted 23 lines · Zhang Jingwang

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 8 lines · Zhang Jingwang
Quoted 4 lines

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

Quoted 7 lines

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Quoted 4 lines
Quoted 1 line

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will not help.

Quoted 7 lines

I think what you did is right for the time being and I'll merge it until we have something better.
The state machine should help in this case since it will effectively switch contexts between two tasks rather than blocking synchronously.

Benny

No! it is not. The patch below is based on the old code. If it was done over new code then you would have seen that the pnfs_{write,read}_retry must call _pnfs_return_layout(,NO_WAIT) without waiting because it is called from the nfsiod_workqueue. But if it is not waiting then there is no point in calling pnfs_initiate_{write,read}().

For writes we can safely remove the call, for reads I would need to check what's best to do.

Boaz

#6 lore raw
Quoted 45 lines · Boaz Harrosh
Quoted 29 lines · Benny Halevy
Quoted 23 lines · Zhang Jingwang

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 8 lines · Zhang Jingwang
Quoted 4 lines

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

Quoted 7 lines

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Quoted 4 lines
Quoted 1 line

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will not help.

Quoted 7 lines

I think what you did is right for the time being and I'll merge it until we have something better.
The state machine should help in this case since it will effectively switch contexts between two tasks rather than blocking synchronously.

Benny

No! it is not. The patch below is based on the old code. If it was done over new code then you would have seen that the pnfs_{write,read}_retry must call _pnfs_return_layout(,NO_WAIT) without waiting because it is called from the nfsiod_workqueue. But if it is not waiting then there is no point in calling pnfs_initiate_{write,read}().

Quoted 4 lines

For writes we can safely remove the call, for reads I would need to check what's best to do.

Boaz

I want to summarize. I've looked at the patch Benny has commited in the latest tree, quoted below please see comments. And it will clearly lock once there is a layout to commit, just like the problem we had recently with the return_layout changes. Since now we are calling Layout_commit/return from the nfsiod.

What we could do for now is drop the calls to pnfs_initiate_*. Altogether and let the pages VFS state machines kick in again later. For writes it will work for sure for reads it should be tested. See below in comments on submitted patch:

Quoted 15 lines · Benny Halevy

df90b7b063154a9a603bafe21f0a9a39607d7b3d

 fs/nfs/pnfs.c |   60 ++++++++++++++++++++++++++++++++++++++-------------------
 1 files changed, 40 insertions(+), 20 deletions(-)
fs/nfs/pnfs.c +3 -0
index a095b42..20285bc 100644
@@ -1497,6 +1497,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
  * cleanup.
  */
 static void
+pnfs_write_retry(struct work_struct *work)
+{
+	struct rpc_task *task;

fs/nfs/pnfs.c pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data) · line 1512

Quoted 10 lines
fs/nfs/pnfs.c +10 -0
+	struct nfs_write_data *wdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	wdata = container_of(task, struct nfs_write_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = wdata->args.offset;
+	range.length = wdata->args.count;
+	_pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE, true);

This is on the nfsiod now. If we need a layoutcommit_ctx it will lock

All we need is:

+ _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE, true);
- _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE, false);

fs/nfs/pnfs.c pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data) · line 1672

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+	pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+			    wdata->pdata.call_ops, wdata->pdata.how);

and:

- pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
- wdata->pdata.call_ops, wdata->pdata.how);

The VFS will reissue these writes soon enough

Quoted 32 lines
fs/nfs/pnfs.c +8 -10
+}
+
+static void
 pnfs_writeback_done(struct nfs_write_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1516,16 +1534,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
 	}
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_RW,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE,
-				    true);
-		pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-				    pdata->call_ops, pdata->how);
+		INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);
 	}
 }
 
@@ -1641,6 +1651,24 @@ out:
  * read_pagelist is done
  */
 static void
+pnfs_read_retry(struct work_struct *work)
+{
+	struct rpc_task *task;

fs/nfs/pnfs.c out: · line 1666

Quoted 10 lines
fs/nfs/pnfs.c +10 -0
+	struct nfs_read_data *rdata;
+	struct nfs4_pnfs_layout_segment range;
+
+	dprintk("%s enter\n", __func__);
+	task = container_of(work, struct rpc_task, u.tk_work);
+	rdata = container_of(task, struct nfs_read_data, task);
+	range.iomode = IOMODE_RW;
+	range.offset = rdata->args.offset;
+	range.length = rdata->args.count;
+	_pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE, true);

Same here though less likely since with reads we probably do not have to layoutcommit unless file is double opened and was written to elsewhere.

But needs to be fixed anyway

fs/nfs/pnfs.c out: · line 1826

Quoted 2 lines
fs/nfs/pnfs.c +2 -0
+	pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+			   rdata->pdata.call_ops);

Consider dropping here too, but needs to be tested.

If it does not work then we need to find a way to propagate the -EAGAIN to the upper layer, the VFS will know what to do.

I do think there is a good chance we don't need to do anything.

Quoted 8 lines
fs/nfs/pnfs.c +3 -0
+}
+
+static void
 pnfs_read_done(struct nfs_read_data *data)
 {
 	struct pnfs_call_data *pdata = &data->pdata;
@@ -1648,16 +1676,8 @@ pnfs_read_done(struct nfs_read_data *data)
 	dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1658

Quoted 10 lines
fs/nfs/pnfs.c +0 -8
 
 	if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-		struct nfs4_pnfs_layout_segment range = {
-			.iomode = IOMODE_ANY,
-			.offset = data->args.offset,
-			.length = data->args.count,
-		};
-		dprintk("%s: retrying\n", __func__);
-		_pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE,
-				    true);

What happens now, if we call _pnfs_return_layout(,,,,false) does it still wait on all lseg's ref? maybe just call with false here and drop the call to pnfs_initiate_read.

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1838

Quoted 4 lines
fs/nfs/pnfs.c +2 -2
-		pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-				   pdata->call_ops);
+		INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+		queue_work(nfsiod_workqueue, &data->task.u.tk_work);

Note that now the upper layers see the -EAGAIN before the return was attempted and mark pending, and the reads will keep banging on old layout for a while.

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1683

Quoted 3 lines
fs/nfs/pnfs.c +0 -0
 	}
 }
 

Boaz

A message that is not in the archive (sent off-list or not yet received): AANLkTimhsjIISik5KvAHDwbEWVdU_wrRPepfXYy30Brl-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#8 lore raw
Quoted 7 lines · Zhang Jingwang

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 6 lines · Zhang Jingwang
Quoted 4 lines

These two functions mustn't be called from the same workqueue. Otherwise deadlock may occur. So we schedule the return_layout_barrier to nfsiod. nfsiod may not be a good choice, maybe we should setup a new workqueue to do the job.

Quoted 9 lines
Quoted 2 lines

Please give more information. When does it happen that pnfs_XXX_done will return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

I wish you would send me the real stack trace and not the explanations because some things has changed and I could find a way to solve it with the new code.

Boaz

Quoted 119 lines
Quoted 8 lines

And please rebase that patch on the latest changes to _pnfs_return_layout(). but since in the new code _pnfs_return_layout() must be called with NO_WAIT if called from the nfsiod then you cannot call pnfs_initiate_write/read() right after. For writes you can get by with doing nothing because the write-back thread will kick in soon enough. For reads I'm not sure, you'll need to send me more information, stack trace.

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will not help.

Quoted 105 lines

Boaz

Quoted 86 lines
---
 fs/nfs/pnfs.c |   58 +++++++++++++++++++++++++++++++++++++++-----------------
 1 files changed, 40 insertions(+), 18 deletions(-)
fs/nfs/pnfs.c +38 -13
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata, struct rpc_task *task, void *data)
  * cleanup.
  */
 static void
+pnfs_write_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_write_data *wdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     wdata = container_of(task, struct nfs_write_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = wdata->args.offset;
+     range.length = wdata->args.count;
+     _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+                         wdata->pdata.call_ops, wdata->pdata.how);
+}
+
+static void
 pnfs_writeback_done(struct nfs_write_data *data)
 {
      struct pnfs_call_data *pdata = &data->pdata;
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *data)
      }
 
      if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_RW,
-                     .offset = data->args.offset,
-                     .length = data->args.count,
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_write(data, NFS_CLIENT(data->inode),
-                                 pdata->call_ops, pdata->how);
+             INIT_WORK(&data->task.u.tk_work, pnfs_write_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
      }
 }
 
@@ -1798,6 +1809,24 @@ out:
  * read_pagelist is done
  */
 static void
+pnfs_read_retry(struct work_struct *work)
+{
+     struct rpc_task *task;
+     struct nfs_read_data *rdata;
+     struct nfs4_pnfs_layout_segment range;
+
+     dprintk("%s enter\n", __func__);
+     task = container_of(work, struct rpc_task, u.tk_work);
+     rdata = container_of(task, struct nfs_read_data, task);
+     range.iomode = IOMODE_RW;
+     range.offset = rdata->args.offset;
+     range.length = rdata->args.count;
+     _pnfs_return_layout(rdata->inode, &range, NULL, RETURN_FILE);
+     pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+                        rdata->pdata.call_ops);
+}
+
+static void
 pnfs_read_done(struct nfs_read_data *data)
 {
      struct pnfs_call_data *pdata = &data->pdata;
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)
      dprintk("%s: Begin (status %d)\n", __func__, data->task.tk_status);
 
      if (pnfs_call_done(pdata, &data->task, data) == -EAGAIN) {
-             struct nfs4_pnfs_layout_segment range = {
-                     .iomode = IOMODE_ANY,
-                     .offset = data->args.offset,
-                     .length = data->args.count,

fs/nfs/pnfs.c pnfs_read_done(struct nfs_read_data *data) · line 1841

Quoted 10 lines
fs/nfs/pnfs.c +2 -5
-             };
-             dprintk("%s: retrying\n", __func__);
-             _pnfs_return_layout(data->inode, &range, NULL, RETURN_FILE);
-             pnfs_initiate_read(data, NFS_CLIENT(data->inode),
-                                pdata->call_ops);
+             INIT_WORK(&data->task.u.tk_work, pnfs_read_retry);
+             queue_work(nfsiod_workqueue, &data->task.u.tk_work);
      }
 }
 

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#9 lore raw

2010/5/24 Boaz Harrosh <[email protected]>:

Quoted 4 lines · Zhang Jingwang
Quoted 3 lines

2010/5/17 Boaz Harrosh <[email protected]>:

Quoted 2 lines · Zhang Jingwang
Quoted 1 line

These two functions mustn't be called from the same workqueue. Oth=

erwise

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

deadlock may occur. So we schedule the return_layout_barrier to nf=

siod.

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

nfsiod may not be a good choice, maybe we should setup a new workq=

ueue

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

to do the job.

Please give more information. When does it happen that pnfs_XXX_don=

e will

Quoted 10 lines
Quoted 8 lines
Quoted 1 line

return -EAGAIN?

network error or something else.

Quoted 3 lines

What is the stack trace of the deadlock?

http://linux-nfs.org/pipermail/pnfs/2010-January/009939.html

I wish you would send me the real stack trace and not the explanation=

s

Quoted 1 line

because some things has changed and I could find a way to solve it wi=

th

Quoted 4 lines

the new code.

Boaz

There is stack dump info in the reply to this message, its URL is
http://linux-nfs.org/pipermail/pnfs/2010-January/010014.html

Quoted 1 line
Quoted 1 line
Quoted 1 line

And please rebase that patch on the latest changes to _pnfs_return_=

layout().

Quoted 1 line
Quoted 1 line
Quoted 1 line

but since in the new code _pnfs_return_layout() must be called with=

NO_WAIT

Quoted 1 line
Quoted 1 line
Quoted 1 line

if called from the nfsiod then you cannot call pnfs_initiate_write/=

read() right

Quoted 1 line
Quoted 1 line
Quoted 1 line

after. For writes you can get by with doing nothing because the wri=

te-back

Quoted 1 line
Quoted 1 line
Quoted 1 line

thread will kick in soon enough. For reads I'm not sure, you'll nee=

d to send

Quoted 5 lines
Quoted 5 lines
Quoted 3 lines

me more information, stack trace.

Or you can wait for the new state machine.

I think the reason of this deadlock is that the put and the wait are in the same workqueue and run serially. So the state machine will no=

t

Quoted 8 lines
Quoted 8 lines

help.

Quoted 7 lines

Boaz

Quoted 4 lines

---
=A0fs/nfs/pnfs.c | =A0 58 +++++++++++++++++++++++++++++++++++++++-=

----------------

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

fs/nfs/pnfs.c pnfs_call_done(struct pnfs_call_data *pdata=

Quoted 7 lines

=A01 files changed, 40 insertions(+), 18 deletions(-)

fs/nfs/pnfs.c +0 -0
index 5ad7fc6..d2b72be 100644
@@ -1655,6 +1655,24 @@ pnfs_call_done(struct pnfs_call_data *pdata=

, struct rpc_task *task, void *data)

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

=A0 * cleanup.
=A0 */
=A0static void
+pnfs_write_retry(struct work_struct *work)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

fs/nfs/pnfs.c

Quoted 8 lines
fs/nfs/pnfs.c +8 -0
+{
+ =A0 =A0 struct rpc_task *task;
+ =A0 =A0 struct nfs_write_data *wdata;
+ =A0 =A0 struct nfs4_pnfs_layout_segment range;
+
+ =A0 =A0 dprintk("%s enter\n", __func__);
+ =A0 =A0 task =3D container_of(work, struct rpc_task, u.tk_work);
+ =A0 =A0 wdata =3D container_of(task, struct nfs_write_data, task=

);

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

+ =A0 =A0 range.iomode =3D IOMODE_RW;
+ =A0 =A0 range.offset =3D wdata->args.offset;
+ =A0 =A0 range.length =3D wdata->args.count;
+ =A0 =A0 _pnfs_return_layout(wdata->inode, &range, NULL, RETURN_F=

ILE);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ =A0 =A0 pnfs_initiate_write(wdata, NFS_CLIENT(wdata->inode),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 wdata->pdata.cal=

l_ops, wdata->pdata.how);

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

fs/nfs/pnfs.c pnfs_writeback_done(struct nfs_write_data *=

Quoted 7 lines
fs/nfs/pnfs.c +3 -0
+}
+
+static void

=A0pnfs_writeback_done(struct nfs_write_data *data) =A0{
=A0 =A0 =A0 struct pnfs_call_data *pdata =3D &data->pdata;

fs/nfs/pnfs.c +0 -0
@@ -1674,15 +1692,8 @@ pnfs_writeback_done(struct nfs_write_data *=

data)

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
fs/nfs/pnfs.c +0 -0
 =A0 =A0 =A0 }
 

=A0 =A0 =A0 if (pnfs_call_done(pdata, &data->task, data) =3D=3D -E=

AGAIN) {

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 struct nfs4_pnfs_layout_segment range =3D=

{

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .iomode =3D IOMODE_RW,
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .offset =3D data->args.o=

ffset,

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .length =3D data->args.c=

ount,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

- =A0 =A0 =A0 =A0 =A0 =A0 };
- =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s: retrying\n", __func__);
- =A0 =A0 =A0 =A0 =A0 =A0 _pnfs_return_layout(data->inode, &range,=

NULL, RETURN_FILE);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 pnfs_initiate_write(data, NFS_CLIENT(dat=

a->inode),

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =

pdata->call_ops, pdata->how);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 INIT_WORK(&data->task.u.tk_work, pnfs_wr=

ite_retry);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 queue_work(nfsiod_workqueue, &data->task=

=2Eu.tk_work);

Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Quoted 7 lines

=A0 =A0 =A0 }
=A0}

fs/nfs/pnfs.c +0 -0
@@ -1798,6 +1809,24 @@ out:

=A0 * read_pagelist is done
=A0 */
=A0static void

fs/nfs/pnfs.c

Quoted 9 lines
fs/nfs/pnfs.c +9 -0
+pnfs_read_retry(struct work_struct *work)
+{
+ =A0 =A0 struct rpc_task *task;
+ =A0 =A0 struct nfs_read_data *rdata;
+ =A0 =A0 struct nfs4_pnfs_layout_segment range;
+
+ =A0 =A0 dprintk("%s enter\n", __func__);
+ =A0 =A0 task =3D container_of(work, struct rpc_task, u.tk_work);
+ =A0 =A0 rdata =3D container_of(task, struct nfs_read_data, task)=

;

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

+ =A0 =A0 range.iomode =3D IOMODE_RW;
+ =A0 =A0 range.offset =3D rdata->args.offset;
+ =A0 =A0 range.length =3D rdata->args.count;
+ =A0 =A0 _pnfs_return_layout(rdata->inode, &range, NULL, RETURN_F=

ILE);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

+ =A0 =A0 pnfs_initiate_read(rdata, NFS_CLIENT(rdata->inode),
+ =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0rdata->pdata.call=

_ops);

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
fs/nfs/pnfs.c +3 -0
+}
+
+static void

=A0pnfs_read_done(struct nfs_read_data *data) =A0{
=A0 =A0 =A0 struct pnfs_call_data *pdata =3D &data->pdata;

fs/nfs/pnfs.c +0 -0
@@ -1805,15 +1834,8 @@ pnfs_read_done(struct nfs_read_data *data)

=A0 =A0 =A0 dprintk("%s: Begin (status %d)\n", __func__, data->tas=

k.tk_status);

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

=A0 =A0 =A0 if (pnfs_call_done(pdata, &data->task, data) =3D=3D -E=

AGAIN) {

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 struct nfs4_pnfs_layout_segment range =3D=

{

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .iomode =3D IOMODE_ANY,
- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .offset =3D data->args.o=

ffset,

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 .length =3D data->args.c=

ount,

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

- =A0 =A0 =A0 =A0 =A0 =A0 };
- =A0 =A0 =A0 =A0 =A0 =A0 dprintk("%s: retrying\n", __func__);
- =A0 =A0 =A0 =A0 =A0 =A0 _pnfs_return_layout(data->inode, &range,=

NULL, RETURN_FILE);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 pnfs_initiate_read(data, NFS_CLIENT(data=

->inode),

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

- =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0p=

data->call_ops);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 INIT_WORK(&data->task.u.tk_work, pnfs_re=

ad_retry);

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

+ =A0 =A0 =A0 =A0 =A0 =A0 queue_work(nfsiod_workqueue, &data->task=

=2Eu.tk_work);

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines

=A0 =A0 =A0 }
=A0}

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs=

" in

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

the body of a message to [email protected]
More majordomo info at =A0http://vger.kernel.org/majordomo-info.htm=

l

Quoted 6 lines
Quoted 4 lines
Quoted 1 line

--=20
Zhang Jingwang
National Research Centre for High Performance Computers Institute of Computing Technology, Chinese Academy of Sciences No. 6, South Kexueyuan Road, Haidian District Beijing, China