pnfs block layout driver based on v3.0-rc2

From: Peng Tao <[email protected]>

Some layout driver like block will have multiple segments. Generic code should be able to handle it.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/pnfs.c |   17 +++++++++++++----
 fs/nfs/pnfs.h |    1 +

 2 files changed, 14 insertions(+), 4 deletions(-)

fs/nfs/pnfs.c +0 -0

index e3d618b..f03a5e0 100644

               if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && is_matching_lseg(&lseg->pls_range, range)) { ret = get_lseg(lseg);

fs/nfs/pnfs.c +2 -0

@@ -1193,10 +1193,18 @@ pnfs_try_to_read_data(struct nfs_read_data *rdata,
  static struct pnfs_layout_segment *pnfs_list_write_lseg(struct inode *inode)
  {
        struct pnfs_layout_segment *lseg, *rv = NULL;
+       loff_t max_pos = 0;
+

fs/nfs/pnfs.c +1 -3

-       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list)
-               if (lseg->pls_range.iomode == IOMODE_RW)
-                       rv = lseg;
        return rv;
  }
 

@@ -1211,6 +1219,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
        if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
                /* references matched in nfs4_layoutcommit_release */
                get_lseg(wdata->lseg);
+               set_bit(NFS_LSEG_LAYOUTCOMMIT, &wdata->lseg->pls_flags);

               wdata->lseg->pls_lc_cred = get_rpccred(wdata->args.context->state->owner->so_cred); mark_as_dirty = true;

fs/nfs/pnfs.h +0 -0

index b071b56..a3fc0f2 100644
@@ -36,6 +36,7 @@
  enum {

       NFS_LSEG_VALID = 0,     /* cleared when lseg is recalled/returned */ NFS_LSEG_ROC,           /* roc bit received from server */
+       NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for layoutcommit */ };

 struct pnfs_layout_segment {
--
1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:37 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 03/34] pnfs: let layoutcommit code handle multiple segments

Quoted 18 lines · Jim Rees

From: Peng Tao <[email protected]>

Some layout driver like block will have multiple segments. Generic code should be able to handle it.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/pnfs.c |   17 +++++++++++++----
 fs/nfs/pnfs.h |    1 +

 2 files changed, 14 insertions(+), 4 deletions(-)

fs/nfs/pnfs.c +0 -0

index e3d618b..f03a5e0 100644
@@ -892,7 +892,7 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,
        dprintk("%s:Begin\n", __func__);
 

fs/nfs/pnfs.c pnfs_find_lseg(struct pnfs_layout_hdr *lo, · line 895

Quoted 3 lines

fs/nfs/pnfs.c +1 -1

        assert_spin_locked(&lo->plh_inode->i_lock);
-       list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
+       list_for_each_entry_reverse(lseg, &lo->plh_segs, pls_list) {

This is a sortred list, and the order of search matters. You can't just reverse it here.

Quoted 5 lines

               if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && is_matching_lseg(&lseg->pls_range, range)) { ret = get_lseg(lseg);

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1202

Quoted 10 lines

fs/nfs/pnfs.c +7 -0

@@ -1193,10 +1193,18 @@ pnfs_try_to_read_data(struct nfs_read_data *rdata,
  static struct pnfs_layout_segment *pnfs_list_write_lseg(struct inode *inode)
  {
        struct pnfs_layout_segment *lseg, *rv = NULL;
+       loff_t max_pos = 0;
+
+       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
+               if (lseg->pls_range.iomode == IOMODE_RW) {
+                       if (max_pos < lseg->pls_end_pos)
+                               max_pos = lseg->pls_end_pos;
+                       if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT,

&lseg->pls_flags))

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1207

Quoted 5 lines

fs/nfs/pnfs.c +4 -0

+                               rv = lseg;
+               }
+       }
+       rv->pls_end_pos = max_pos;
 

The idea here was that it could be extended to use segment by returning a list of affected lsegs,
not so,e random one. Because otherwise you have problems with the fact that relevant but not
returned lsegs are going to get there refcounts messed up.

Fred

fs/nfs/pnfs.c pnfs_set_layoutcommit(struct nfs_write_data *wdata) · line 1222

Quoted 11 lines

fs/nfs/pnfs.c +1 -3

-       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list)
-               if (lseg->pls_range.iomode == IOMODE_RW)
-                       rv = lseg;
        return rv;
  }
 

@@ -1211,6 +1219,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
        if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
                /* references matched in nfs4_layoutcommit_release */
                get_lseg(wdata->lseg);
+               set_bit(NFS_LSEG_LAYOUTCOMMIT,

&wdata->lseg->pls_flags);

Quoted 10 lines

               wdata->lseg->pls_lc_cred = get_rpccred(wdata->args.context->state->owner- so_cred);
               mark_as_dirty = true;

fs/nfs/pnfs.h +0 -0

index b071b56..a3fc0f2 100644
@@ -36,6 +36,7 @@
  enum {

Quoted 3 lines

       NFS_LSEG_VALID = 0,     /* cleared when lseg is recalled/returned */ NFS_LSEG_ROC,           /* roc bit received from server */
+       NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for

layoutcommit */

Quoted 11 lines

 };

 struct pnfs_layout_segment {
--
1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Hi, Fred,

Quoted 27 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:37 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 03/34] pnfs: let layoutcommit code handle multiple segments

Quoted 18 lines · Jim Rees

From: Peng Tao <[email protected]>

Some layout driver like block will have multiple segments. Generic code should be able to handle it.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/pnfs.c |   17 +++++++++++++----
 fs/nfs/pnfs.h |    1 +

 2 files changed, 14 insertions(+), 4 deletions(-)

fs/nfs/pnfs.c +0 -0

index e3d618b..f03a5e0 100644
@@ -892,7 +892,7 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,
        dprintk("%s:Begin\n", __func__);
 

Quoted 6 lines

fs/nfs/pnfs.c pnfs_find_lseg(struct pnfs_layout_hdr *lo, · line 895

Quoted 3 lines

fs/nfs/pnfs.c +1 -1

        assert_spin_locked(&lo->plh_inode->i_lock);
-       list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
+       list_for_each_entry_reverse(lseg, &lo->plh_segs, pls_list) {

This is a sortred list, and the order of search matters.  You can't just reverse it here.

The layout segment list is in offset increasing order. But the lookup code here assumes it's a decreasing ordered list.
To fix it, we should either reverse lookup the list, or change the break condition test. Otherwise lookup always fails if not matching the first one.

Quoted 23 lines

Quoted 6 lines

               if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && is_matching_lseg(&lseg->pls_range, range)) { ret = get_lseg(lseg);

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1202

Quoted 10 lines

fs/nfs/pnfs.c +7 -0

@@ -1193,10 +1193,18 @@ pnfs_try_to_read_data(struct nfs_read_data *rdata,
  static struct pnfs_layout_segment *pnfs_list_write_lseg(struct inode *inode)
  {
        struct pnfs_layout_segment *lseg, *rv = NULL;
+       loff_t max_pos = 0;
+
+       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
+               if (lseg->pls_range.iomode == IOMODE_RW) {
+                       if (max_pos < lseg->pls_end_pos)
+                               max_pos = lseg->pls_end_pos;
+                       if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT,

&lseg->pls_flags))

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1207

Quoted 5 lines

fs/nfs/pnfs.c +4 -0

+                               rv = lseg;
+               }
+       }
+       rv->pls_end_pos = max_pos;
 

Quoted 5 lines

The idea here was that it could be extended to use segment by returning a list of affected lsegs,
not so,e random one.  Because otherwise you have problems with the fact that relevant but not
returned lsegs are going to get there refcounts messed up.

The above code relies on NFS_INO_LAYOUTCOMMIT bit to ensure that only one inode lseg has NFS_LSEG_LAYOUTCOMMIT set. But, you are right. The layoutcommit code needs a second thought. How about making it return a list of affected lsegs and pass them around layoutcommit_procs?

Thanks,
Tao

Quoted 40 lines

Fred

fs/nfs/pnfs.c pnfs_set_layoutcommit(struct nfs_write_data *wdata) · line 1222

Quoted 11 lines

fs/nfs/pnfs.c +1 -3

-       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list)
-               if (lseg->pls_range.iomode == IOMODE_RW)
-                       rv = lseg;
        return rv;
  }
 

@@ -1211,6 +1219,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
        if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
                /* references matched in nfs4_layoutcommit_release */
                get_lseg(wdata->lseg);
+               set_bit(NFS_LSEG_LAYOUTCOMMIT,

&wdata->lseg->pls_flags);

Quoted 10 lines

               wdata->lseg->pls_lc_cred = get_rpccred(wdata->args.context->state->owner- so_cred);
               mark_as_dirty = true;

fs/nfs/pnfs.h +0 -0

index b071b56..a3fc0f2 100644
@@ -36,6 +36,7 @@
  enum {

Quoted 3 lines

       NFS_LSEG_VALID = 0,     /* cleared when lseg is recalled/returned */ NFS_LSEG_ROC,           /* roc bit received from server */
+       NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for

layoutcommit */

Quoted 11 lines

 };

 struct pnfs_layout_segment {
--
1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 4 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Hi, Fred,

Quoted 27 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:37 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 03/34] pnfs: let layoutcommit code handle multiple segments

Quoted 18 lines · Jim Rees

From: Peng Tao <[email protected]>

Some layout driver like block will have multiple segments. Generic code should be able to handle it.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/pnfs.c |   17 +++++++++++++----
 fs/nfs/pnfs.h |    1 +
 2 files changed, 14 insertions(+), 4 deletions(-)

fs/nfs/pnfs.c +0 -0

index e3d618b..f03a5e0 100644
@@ -892,7 +892,7 @@ pnfs_find_lseg(struct pnfs_layout_hdr *lo,
       dprintk("%s:Begin\n", __func__);
 

Quoted 6 lines

fs/nfs/pnfs.c pnfs_find_lseg(struct pnfs_layout_hdr *lo, · line 895

Quoted 3 lines

fs/nfs/pnfs.c +1 -1

       assert_spin_locked(&lo->plh_inode->i_lock);
-       list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
+       list_for_each_entry_reverse(lseg, &lo->plh_segs, pls_list) {

This is a sortred list, and the order of search matters. You can't just reverse it here.

The layout segment list is in offset increasing order. But the lookup code here assumes it's a decreasing ordered list.
To fix it, we should either reverse lookup the list, or change the break condition test. Otherwise lookup always fails if not matching the first one.

Quoted 22 lines

Quoted 5 lines

fs/nfs/pnfs.c +0 -0

               if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
                   is_matching_lseg(&lseg->pls_range, range)) {
                       ret = get_lseg(lseg);

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1202

Quoted 10 lines

fs/nfs/pnfs.c +7 -0

@@ -1193,10 +1193,18 @@ pnfs_try_to_read_data(struct nfs_read_data *rdata,
 static struct pnfs_layout_segment *pnfs_list_write_lseg(struct inode *inode)
 {
       struct pnfs_layout_segment *lseg, *rv = NULL;
+       loff_t max_pos = 0;
+
+       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
+               if (lseg->pls_range.iomode == IOMODE_RW) {
+                       if (max_pos < lseg->pls_end_pos)
+                               max_pos = lseg->pls_end_pos;
+                       if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT,

&lseg->pls_flags))

fs/nfs/pnfs.c pnfs_try_to_read_data(struct nfs_read_data *rdata, · line 1207

Quoted 5 lines

fs/nfs/pnfs.c +4 -0

+                               rv = lseg;
+               }
+       }
+       rv->pls_end_pos = max_pos;
 

Quoted 5 lines

The idea here was that it could be extended to use segment by returning a list of affected lsegs,
not so,e random one. Because otherwise you have problems with the fact that relevant but not
returned lsegs are going to get there refcounts messed up.

The above code relies on NFS_INO_LAYOUTCOMMIT bit to ensure that only one inode lseg has NFS_LSEG_LAYOUTCOMMIT set. But, you are right. The layoutcommit code needs a second thought. How about making it return a list of affected lsegs and pass them around layoutcommit_procs?

Thanks,
Tao

Quoted 40 lines

Fred

fs/nfs/pnfs.c pnfs_set_layoutcommit(struct nfs_write_data *wdata) · line 1222

Quoted 11 lines

fs/nfs/pnfs.c +1 -3

-       list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list)
-               if (lseg->pls_range.iomode == IOMODE_RW)
-                       rv = lseg;
       return rv;
 }
 

@@ -1211,6 +1219,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
       if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
               /* references matched in nfs4_layoutcommit_release */
               get_lseg(wdata->lseg);
+               set_bit(NFS_LSEG_LAYOUTCOMMIT,

&wdata->lseg->pls_flags);

fs/nfs/pnfs.h line 39

Quoted 13 lines

fs/nfs/pnfs.c +0 -0

               wdata->lseg->pls_lc_cred =
                       get_rpccred(wdata->args.context->state->owner-
 so_cred);

               mark_as_dirty = true;

fs/nfs/pnfs.h +1 -0

index b071b56..a3fc0f2 100644
@@ -36,6 +36,7 @@
 enum {
       NFS_LSEG_VALID = 0,     /* cleared when lseg is recalled/returned */
       NFS_LSEG_ROC,           /* roc bit received from server */
+       NFS_LSEG_LAYOUTCOMMIT,  /* layoutcommit bit set for

layoutcommit */

Quoted 11 lines

fs/nfs/pnfs.h +0 -1

 };
 
 struct pnfs_layout_segment {
--

1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 4 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

From: Peng Tao <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Reported-by Alexandros Batsakis <[email protected]>

Signed-off-by Andy Adamson <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---

 fs/nfs/file.c          |   26 ++++++++++-
 fs/nfs/pnfs.c          |   41 +++++++++++++++++
 fs/nfs/pnfs.h          |  115 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/write.c         |   12 +++--
 include/linux/nfs_fs.h |    3 +-
 5 files changed, 189 insertions(+), 8 deletions(-)

fs/nfs/file.c +0 -0

index 2f093ed..1768762 100644
@@ -384,12 +384,15 @@ static int nfs_write_begin(struct file *file, struct address_space *mapping,
        pgoff_t index = pos >> PAGE_CACHE_SHIFT;
        struct page *page;
        int once_thru = 0;

 start:
       /*
        * Prevent starvation issues if someone is doing a consistency

fs/nfs/file.c +18 -1

@@ -409,6 +412,9 @@ start:
        if (ret) {
                unlock_page(page);
                page_cache_release(page);
+               *pagep = NULL;
+               *fsdata = NULL;
+               goto out;
        } else if (!once_thru &&
                   nfs_want_read_modify_write(file, page, pos, len)) {
                once_thru = 1;

@@ -417,6 +423,12 @@ start:
                if (!ret)
                        goto start;
        }
+       ret = pnfs_write_begin(file, page, pos, len, lseg, fsdata);
+ out:
+       if (ret) {
+               put_lseg(lseg);
+               *fsdata = NULL;
+       }
        return ret;
  }
 

@@ -426,6 +438,7 @@ static int nfs_write_end(struct file *file, struct address_space *mapping,
  {
        unsigned offset = pos & (PAGE_CACHE_SIZE - 1);
        int status;
+       struct pnfs_layout_segment *lseg;
 
        dfprintk(PAGECACHE, "NFS: write_end(%s/%s(%ld), %u@%lld)\n",
                file->f_path.dentry->d_parent->d_name.name,

@@ -452,10 +465,17 @@ static int nfs_write_end(struct file *file, struct address_space *mapping,
                        zero_user_segment(page, pglen, PAGE_CACHE_SIZE);
        }
 
-       status = nfs_updatepage(file, page, offset, copied);
+       lseg = nfs4_pull_lseg_from_fsdata(file, fsdata);
+       status = pnfs_write_end(file, page, pos, len, copied, lseg);
+       if (status)
+               goto out;
+       status = nfs_updatepage(file, page, offset, copied, lseg, fsdata);
 
+out:
        unlock_page(page);
        page_cache_release(page);
+       pnfs_write_end_cleanup(file, fsdata);
+       put_lseg(lseg);
 
        if (status < 0)
                return status;

@@ -577,7 +597,7 @@ static int nfs_vm_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
 
        ret = VM_FAULT_LOCKED;

       if (nfs_flush_incompatible(filp, page) == 0 &&
-           nfs_updatepage(filp, page, 0, pagelen) == 0)
+           nfs_updatepage(filp, page, 0, pagelen, NULL, NULL) == 0) goto out;

       ret = VM_FAULT_SIGBUS;

fs/nfs/pnfs.c +35 -0

index f03a5e0..e693718 100644
@@ -1138,6 +1138,41 @@ pnfs_try_to_write_data(struct nfs_write_data *wdata,
  }
 
  /*
+ * This gives the layout driver an opportunity to read in page "around"
+ * the data to be written.  It returns 0 on success, otherwise an error code
+ * which will either be passed up to user, or ignored if
+ * some previous part of write succeeded.
+ * Note the range [pos, pos+len-1] is entirely within the page.
+ */
+int _pnfs_write_begin(struct inode *inode, struct page *page,
+                     loff_t pos, unsigned len,
+                     struct pnfs_layout_segment *lseg,
+                     struct pnfs_fsdata **fsdata)
+{
+       struct pnfs_fsdata *data;
+       int status = 0;
+
+       dprintk("--> %s: pos=%llu len=%u\n",
+               __func__, (unsigned long long)pos, len);
+       data = kzalloc(sizeof(struct pnfs_fsdata), GFP_KERNEL);
+       if (!data) {
+               status = -ENOMEM;
+               goto out;
+       }
+       data->lseg = lseg; /* refcount passed into data to be managed there */
+       status = NFS_SERVER(inode)->pnfs_curr_ld->write_begin(
+                                               lseg, page, pos, len, data);
+       if (status) {
+               kfree(data);
+               data = NULL;
+       }
+out:
+       *fsdata = data;
+       dprintk("<-- %s: status=%d\n", __func__, status);
+       return status;
+}
+
+/*
  * Called by non rpc-based layout drivers
  */

 int

fs/nfs/pnfs.c +6 -0

@@ -1237,6 +1272,12 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
  }
  EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
+{
+       /* lseg refcounting handled directly in nfs_write_end */
+       kfree(fsdata);
+}
+
  /*
  * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
  * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough

fs/nfs/pnfs.h +26 -0

index a3fc0f2..525ec55 100644
@@ -54,6 +54,12 @@ enum pnfs_try_status {
        PNFS_NOT_ATTEMPTED = 1,
  };
 
+struct pnfs_fsdata {
+       struct pnfs_layout_segment *lseg;
+       int bypass_eof;
+       void *private;
+};
+
  #ifdef CONFIG_NFS_V4_1
 
  #define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4"

@@ -106,6 +112,14 @@ struct pnfs_layoutdriver_type {
         */
        enum pnfs_try_status (*read_pagelist) (struct nfs_read_data *nfs_data);
        enum pnfs_try_status (*write_pagelist) (struct nfs_write_data *nfs_data, int how);
+       int (*write_begin) (struct pnfs_layout_segment *lseg, struct page *page,
+                           loff_t pos, unsigned count,
+                           struct pnfs_fsdata *fsdata);
+       int (*write_end)(struct inode *inode, struct page *page, loff_t pos,
+                        unsigned count, unsigned copied,
+                        struct pnfs_layout_segment *lseg);
+       void (*write_end_cleanup)(struct file *filp,
+                                 struct pnfs_fsdata *fsdata);
 
        void (*free_deviceid_node) (struct nfs4_deviceid_node *);
 

@@ -175,6 +189,7 @@ enum pnfs_try_status pnfs_try_to_write_data(struct nfs_write_data *,
  enum pnfs_try_status pnfs_try_to_read_data(struct nfs_read_data *,
                                            const struct rpc_call_ops *);
  bool pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, struct nfs_page *req);
+void pnfs_free_fsdata(struct pnfs_fsdata *fsdata);
  int pnfs_layout_process(struct nfs4_layoutget *lgp);
  void pnfs_free_lseg_list(struct list_head *tmp_list);
  void pnfs_destroy_layout(struct nfs_inode *);

@@ -186,6 +201,10 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
  int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
                                  struct pnfs_layout_hdr *lo,
                                  struct nfs4_state *open_state);
+int _pnfs_write_begin(struct inode *inode, struct page *page,
+                     loff_t pos, unsigned len,
+                     struct pnfs_layout_segment *lseg,
+                     struct pnfs_fsdata **fsdata);
  int mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
                                struct list_head *tmp_list,
                                struct pnfs_layout_range *recall_range);

@@ -287,6 +306,13 @@ static inline void pnfs_clear_request_commit(struct nfs_page *req)
                put_lseg(req->wb_commit_lseg);
  }
 
+static inline int pnfs_grow_ok(struct pnfs_layout_segment *lseg,
+                              struct pnfs_fsdata *fsdata)
+{
+       return !fsdata  || ((struct pnfs_layout_segment *)fsdata == lseg) ||
+               !fsdata->bypass_eof;
+}
+
  /* Should the pNFS client commit and return the layout upon a setattr */

 static inline bool pnfs_ld_layoutret_on_setattr(struct inode *inode)

fs/nfs/pnfs.h +62 -0

@@ -297,6 +323,49 @@ pnfs_ld_layoutret_on_setattr(struct inode *inode)
                PNFS_LAYOUTRET_ON_SETATTR;
  }
 
+static inline int pnfs_write_begin(struct file *filp, struct page *page,
+                                  loff_t pos, unsigned len,
+                                  struct pnfs_layout_segment *lseg,
+                                  void **fsdata)
+{
+       struct inode *inode = filp->f_dentry->d_inode;
+       struct nfs_server *nfss = NFS_SERVER(inode);
+       int status = 0;
+
+       *fsdata = lseg;
+       if (lseg && nfss->pnfs_curr_ld->write_begin)
+               status = _pnfs_write_begin(inode, page, pos, len, lseg,
+                                          (struct pnfs_fsdata **) fsdata);
+       return status;
+}
+
+/* CAREFUL - what happens if copied < len??? */
+static inline int pnfs_write_end(struct file *filp, struct page *page,
+                                loff_t pos, unsigned len, unsigned copied,
+                                struct pnfs_layout_segment *lseg)
+{
+       struct inode *inode = filp->f_dentry->d_inode;
+       struct nfs_server *nfss = NFS_SERVER(inode);
+
+       if (nfss->pnfs_curr_ld && nfss->pnfs_curr_ld->write_end)
+               return nfss->pnfs_curr_ld->write_end(inode, page, pos, len,
+                                                    copied, lseg);
+       else
+               return 0;
+}
+
+static inline void pnfs_write_end_cleanup(struct file *filp, void *fsdata)
+{
+       struct nfs_server *nfss = NFS_SERVER(filp->f_dentry->d_inode);
+
+       if (fsdata && nfss->pnfs_curr_ld) {
+               if (nfss->pnfs_curr_ld->write_end_cleanup)
+                       nfss->pnfs_curr_ld->write_end_cleanup(filp, fsdata);
+               if (nfss->pnfs_curr_ld->write_begin)
+                       pnfs_free_fsdata(fsdata);
+       }
+}
+
  static inline int pnfs_return_layout(struct inode *ino)
  {
        struct nfs_inode *nfsi = NFS_I(ino);

@@ -317,6 +386,19 @@ static inline void pnfs_pageio_init(struct nfs_pageio_descriptor *pgio,
                pgio->pg_test = ld->pg_test;
  }
 
+static inline struct pnfs_layout_segment *
+nfs4_pull_lseg_from_fsdata(struct file *filp, void *fsdata)
+{
+       if (fsdata) {
+               struct nfs_server *nfss = NFS_SERVER(filp->f_dentry->d_inode);
+
+               if (nfss->pnfs_curr_ld && nfss->pnfs_curr_ld->write_begin)
+                       return ((struct pnfs_fsdata *) fsdata)->lseg;
+               return (struct pnfs_layout_segment *)fsdata;
+       }
+       return NULL;
+}
+
  #else  /* CONFIG_NFS_V4_1 */
 
  static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)

@@ -345,6 +427,12 @@ pnfs_update_layout(struct inode *ino, struct nfs_open_context *ctx,
        return NULL;
  }
 
+static inline int pnfs_grow_ok(struct pnfs_layout_segment *lseg,
+                              struct pnfs_fsdata *fsdata)
+{
+       return 1;
+}
+

 static inline enum pnfs_try_status pnfs_try_to_read_data(struct nfs_read_data *data, const struct rpc_call_ops *call_ops)

fs/nfs/pnfs.h +20 -0

@@ -364,6 +452,26 @@ static inline int pnfs_return_layout(struct inode *ino)
        return 0;
  }
 
+static inline int pnfs_write_begin(struct file *filp, struct page *page,
+                                  loff_t pos, unsigned len,
+                                  struct pnfs_layout_segment *lseg,
+                                  void **fsdata)
+{
+       *fsdata = NULL;
+       return 0;
+}
+
+static inline int pnfs_write_end(struct file *filp, struct page *page,
+                                loff_t pos, unsigned len, unsigned copied,
+                                struct pnfs_layout_segment *lseg)
+{
+       return 0;
+}
+
+static inline void pnfs_write_end_cleanup(struct file *filp, void *fsdata)
+{
+}
+

 static inline bool pnfs_ld_layoutret_on_setattr(struct inode *inode) {

fs/nfs/pnfs.h +7 -0

@@ -435,6 +543,13 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync)
  static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
  {
  }
+
+static inline struct pnfs_layout_segment *
+nfs4_pull_lseg_from_fsdata(struct file *filp, void *fsdata)
+{
+       return NULL;
+}
+
  #endif /* CONFIG_NFS_V4_1 */
 
  #endif /* FS_NFS_PNFS_H */

fs/nfs/write.c +8 -4

index e268e3b..75e2a6b 100644
@@ -673,7 +673,9 @@ out:
  }
 
  static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
-               unsigned int offset, unsigned int count)
+               unsigned int offset, unsigned int count,
+               struct pnfs_layout_segment *lseg, void *fsdata)
+
  {
        struct nfs_page *req;
 

@@ -681,7 +683,8 @@ static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
        if (IS_ERR(req))
                return PTR_ERR(req);
        /* Update file length */
-       nfs_grow_file(page, offset, count);
+       if (pnfs_grow_ok(lseg, fsdata))
+               nfs_grow_file(page, offset, count);
        nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
        nfs_mark_request_dirty(req);
        nfs_clear_page_tag_locked(req);

@@ -734,7 +737,8 @@ static int nfs_write_pageuptodate(struct page *page, struct inode *inode)
  * things with a page scheduled for an RPC call (e.g. invalidate it).
  */
  int nfs_updatepage(struct file *file, struct page *page,
-               unsigned int offset, unsigned int count)
+               unsigned int offset, unsigned int count,
+               struct pnfs_layout_segment *lseg, void *fsdata)
  {
        struct nfs_open_context *ctx = nfs_file_open_context(file);
        struct inode    *inode = page->mapping->host;

@@ -759,7 +763,7 @@ int nfs_updatepage(struct file *file, struct page *page,
                offset = 0;
        }
 
-       status = nfs_writepage_setup(ctx, page, offset, count);
+       status = nfs_writepage_setup(ctx, page, offset, count, lseg, fsdata);
        if (status < 0)
                nfs_set_pageerror(page);
 

include/linux/nfs_fs.h +2 -2

index 1b93b9c..be1ac1d 100644
@@ -510,7 +510,8 @@ extern int  nfs_congestion_kb;
  extern int  nfs_writepage(struct page *page, struct writeback_control *wbc);
  extern int  nfs_writepages(struct address_space *, struct writeback_control *);
  extern int  nfs_flush_incompatible(struct file *file, struct page *page);
-extern int  nfs_updatepage(struct file *, struct page *, unsigned int, unsigned int);
+extern int  nfs_updatepage(struct file *, struct page *, unsigned int, unsigned int,
+                       struct pnfs_layout_segment *, void *);
  extern void nfs_writeback_done(struct rpc_task *, struct nfs_write_data *);
 
  /*
--

1.7.4.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:44 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 04/34] pnfs: hook nfs_write_begin/end to allow layout driver manipulation

Quoted 9 lines · Jim Rees

From: Peng Tao <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Reported-by Alexandros Batsakis <[email protected]>

Signed-off-by Andy Adamson <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

Quoted 4 lines

---
 fs/nfs/file.c          |   26 ++++++++++-
 fs/nfs/pnfs.c          |   41 +++++++++++++++++
 fs/nfs/pnfs.h          |  115

++++++++++++++++++++++++++++++++++++++++++++++++

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct

Quoted 9 lines

 fs/nfs/write.c         |   12 +++--
 include/linux/nfs_fs.h |    3 +-
 5 files changed, 189 insertions(+), 8 deletions(-)

fs/nfs/file.c +0 -0

index 2f093ed..1768762 100644
@@ -384,12 +384,15 @@ static int nfs_write_begin(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct address_space *mapping, · line 395

Quoted 13 lines

fs/nfs/file.c +4 -1

       pgoff_t index = pos >> PAGE_CACHE_SHIFT;
       struct page *page;
       int once_thru = 0;
+       struct pnfs_layout_segment *lseg;
 
       dfprintk(PAGECACHE, "NFS: write_begin(%s/%s(%ld), %u@%lld)\n",
               file->f_path.dentry->d_parent->d_name.name,
               file->f_path.dentry->d_name.name,
               mapping->host->i_ino, len, (long long) pos);
-
+       lseg = pnfs_update_layout(mapping->host,
+                                 nfs_file_open_context(file),
+                                 pos, len, IOMODE_RW, GFP_NOFS);

This looks like it is left over from before the rearrangements done to where pnfs_update_layout.
In particular, we don't want to hold the reference on the lseg from here until flush time. And there
seems to be no reason to. If the client needs a layout to deal with read-in here, it should instead
trigger the nfs_want_read_modify_write clause.

Fred

fs/nfs/file.c static int nfs_write_end(struct file *file, struct

Quoted 27 lines

fs/nfs/file.c +9 -0

 start:
       /*
        * Prevent starvation issues if someone is doing a consistency

@@ -409,6 +412,9 @@ start:
       if (ret) {
               unlock_page(page);
               page_cache_release(page);
+               *pagep = NULL;
+               *fsdata = NULL;
+               goto out;
       } else if (!once_thru &&
                  nfs_want_read_modify_write(file, page, pos, len)) {
               once_thru = 1;

@@ -417,6 +423,12 @@ start:
               if (!ret)
                       goto start;
       }
+       ret = pnfs_write_begin(file, page, pos, len, lseg, fsdata);
+ out:
+       if (ret) {
+               put_lseg(lseg);
+               *fsdata = NULL;
+       }
       return ret;
 }
 

@@ -426,6 +438,7 @@ static int nfs_write_end(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_end(struct file *file, struct

Quoted 8 lines

fs/nfs/file.c +1 -0

 {
       unsigned offset = pos & (PAGE_CACHE_SIZE - 1);
       int status;
+       struct pnfs_layout_segment *lseg;
 
       dfprintk(PAGECACHE, "NFS: write_end(%s/%s(%ld), %u@%lld)\n",
               file->f_path.dentry->d_parent->d_name.name,

@@ -452,10 +465,17 @@ static int nfs_write_end(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_end(struct file *file, struct · line 465

Quoted 1 line

fs/nfs/file.c +0 -0

                       zero_user_segment(page, pglen,

PAGE_CACHE_SIZE);

fs/nfs/file.c static int nfs_vm_page_mkwrite(struct vm_area_struct

Quoted 18 lines

fs/nfs/file.c +8 -1

       }
 
-       status = nfs_updatepage(file, page, offset, copied);
+       lseg = nfs4_pull_lseg_from_fsdata(file, fsdata);
+       status = pnfs_write_end(file, page, pos, len, copied, lseg);
+       if (status)
+               goto out;
+       status = nfs_updatepage(file, page, offset, copied, lseg, fsdata);
 
+out:
       unlock_page(page);
       page_cache_release(page);
+       pnfs_write_end_cleanup(file, fsdata);
+       put_lseg(lseg);
 
       if (status < 0)
               return status;

@@ -577,7 +597,7 @@ static int nfs_vm_page_mkwrite(struct vm_area_struct

*vma, struct vm_fault *vmf)

fs/nfs/pnfs.c pnfs_try_to_write_data(struct nfs_write_data

Quoted 13 lines

fs/nfs/file.c +1 -1

       ret = VM_FAULT_LOCKED;
       if (nfs_flush_incompatible(filp, page) == 0 &&
-           nfs_updatepage(filp, page, 0, pagelen) == 0)
+           nfs_updatepage(filp, page, 0, pagelen, NULL, NULL) == 0)
               goto out;
 
       ret = VM_FAULT_SIGBUS;

fs/nfs/pnfs.c +0 -0

index f03a5e0..e693718 100644
@@ -1138,6 +1138,41 @@ pnfs_try_to_write_data(struct nfs_write_data

*wdata,

Quoted 17 lines

fs/nfs/pnfs.c +14 -0

 }
 
 /*
+ * This gives the layout driver an opportunity to read in page "around"
+ * the data to be written.  It returns 0 on success, otherwise an error code
+ * which will either be passed up to user, or ignored if
+ * some previous part of write succeeded.
+ * Note the range [pos, pos+len-1] is entirely within the page.
+ */
+int _pnfs_write_begin(struct inode *inode, struct page *page,
+                     loff_t pos, unsigned len,
+                     struct pnfs_layout_segment *lseg,
+                     struct pnfs_fsdata **fsdata)
+{
+       struct pnfs_fsdata *data;
+       int status = 0;
+

fs/nfs/pnfs.c pnfs_try_to_write_data(struct nfs_write_data *wdata,

Quoted 10 lines

fs/nfs/pnfs.c +10 -0

+       dprintk("--> %s: pos=%llu len=%u\n",
+               __func__, (unsigned long long)pos, len);
+       data = kzalloc(sizeof(struct pnfs_fsdata), GFP_KERNEL);
+       if (!data) {
+               status = -ENOMEM;
+               goto out;
+       }
+       data->lseg = lseg; /* refcount passed into data to be managed there */
+       status = NFS_SERVER(inode)->pnfs_curr_ld->write_begin(
+                                               lseg, page, pos,

len, data);

Quoted 45 lines

fs/nfs/pnfs.c +17 -0

+       if (status) {
+               kfree(data);
+               data = NULL;
+       }
+out:
+       *fsdata = data;
+       dprintk("<-- %s: status=%d\n", __func__, status);
+       return status;
+}
+
+/*
 * Called by non rpc-based layout drivers
 */
 int

@@ -1237,6 +1272,12 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 }
 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
+{
+       /* lseg refcounting handled directly in nfs_write_end */
+       kfree(fsdata);
+}
+
 /*
 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough

fs/nfs/pnfs.h +6 -0

index a3fc0f2..525ec55 100644
@@ -54,6 +54,12 @@ enum pnfs_try_status {
       PNFS_NOT_ATTEMPTED = 1,
 };
 
+struct pnfs_fsdata {
+       struct pnfs_layout_segment *lseg;
+       int bypass_eof;
+       void *private;
+};
+
 #ifdef CONFIG_NFS_V4_1
 
 #define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4"

fs/nfs/pnfs.h struct pnfs_layoutdriver_type { · line 113

Quoted 2 lines

fs/nfs/pnfs.h +0 -0

@@ -106,6 +112,14 @@ struct pnfs_layoutdriver_type {
        */
       enum pnfs_try_status (*read_pagelist) (struct nfs_read_data

*nfs_data);

fs/nfs/pnfs.h struct pnfs_layoutdriver_type { · line 114

Quoted 1 line

fs/nfs/pnfs.h +0 -0

       enum pnfs_try_status (*write_pagelist) (struct nfs_write_data

*nfs_data, int how);

fs/nfs/pnfs.h struct pnfs_layoutdriver_type { · line 115

Quoted 1 line

fs/nfs/pnfs.h +1 -0

+       int (*write_begin) (struct pnfs_layout_segment *lseg, struct page

*page,

fs/nfs/pnfs.h enum pnfs_try_status pnfs_try_to_write_data(struct

Quoted 11 lines

fs/nfs/pnfs.h +7 -0

+                           loff_t pos, unsigned count,
+                           struct pnfs_fsdata *fsdata);
+       int (*write_end)(struct inode *inode, struct page *page, loff_t pos,
+                        unsigned count, unsigned copied,
+                        struct pnfs_layout_segment *lseg);
+       void (*write_end_cleanup)(struct file *filp,
+                                 struct pnfs_fsdata *fsdata);
 
       void (*free_deviceid_node) (struct nfs4_deviceid_node *);
 

@@ -175,6 +189,7 @@ enum pnfs_try_status pnfs_try_to_write_data(struct

nfs_write_data *,

fs/nfs/pnfs.h enum pnfs_try_status pnfs_try_to_write_data(struct · line 190

Quoted 2 lines

fs/nfs/pnfs.h +0 -0

 enum pnfs_try_status pnfs_try_to_read_data(struct nfs_read_data *,
                                           const struct

rpc_call_ops *);

fs/nfs/pnfs.h enum pnfs_try_status pnfs_try_to_write_data(struct · line 191

Quoted 1 line

fs/nfs/pnfs.h +0 -0

 bool pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page

*prev, struct nfs_page *req);

Quoted 5 lines

fs/nfs/pnfs.h +1 -0

+void pnfs_free_fsdata(struct pnfs_fsdata *fsdata);
 int pnfs_layout_process(struct nfs4_layoutget *lgp);
 void pnfs_free_lseg_list(struct list_head *tmp_list);
 void pnfs_destroy_layout(struct nfs_inode *);

fs/nfs/pnfs.h void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, · line 210

Quoted 10 lines

fs/nfs/pnfs.h +4 -0

@@ -186,6 +201,10 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
 int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
                                 struct pnfs_layout_hdr *lo,
                                 struct nfs4_state *open_state);
+int _pnfs_write_begin(struct inode *inode, struct page *page,
+                     loff_t pos, unsigned len,
+                     struct pnfs_layout_segment *lseg,
+                     struct pnfs_fsdata **fsdata);
 int mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
                               struct list_head *tmp_list,
                               struct pnfs_layout_range

*recall_range);

fs/nfs/pnfs.h static inline void pnfs_clear_request_commit(struct

Quoted 1 line

fs/nfs/pnfs.h +0 -0

@@ -287,6 +306,13 @@ static inline void pnfs_clear_request_commit(struct

nfs_page *req)

Quoted 20 lines

fs/nfs/pnfs.h +10 -0

               put_lseg(req->wb_commit_lseg);
 }
 
+static inline int pnfs_grow_ok(struct pnfs_layout_segment *lseg,
+                              struct pnfs_fsdata *fsdata)
+{
+       return !fsdata  || ((struct pnfs_layout_segment *)fsdata == lseg) ||
+               !fsdata->bypass_eof;
+}
+
 /* Should the pNFS client commit and return the layout upon a setattr */
 static inline bool
 pnfs_ld_layoutret_on_setattr(struct inode *inode)

@@ -297,6 +323,49 @@ pnfs_ld_layoutret_on_setattr(struct inode *inode)
               PNFS_LAYOUTRET_ON_SETATTR;
 }
 
+static inline int pnfs_write_begin(struct file *filp, struct page *page,
+                                  loff_t pos, unsigned len,
+                                  struct pnfs_layout_segment *lseg,

fs/nfs/pnfs.h pnfs_ld_layoutret_on_setattr(struct inode *inode) · line 338

Quoted 10 lines

fs/nfs/pnfs.h +10 -0

+                                  void **fsdata)
+{
+       struct inode *inode = filp->f_dentry->d_inode;
+       struct nfs_server *nfss = NFS_SERVER(inode);
+       int status = 0;
+
+       *fsdata = lseg;
+       if (lseg && nfss->pnfs_curr_ld->write_begin)
+               status = _pnfs_write_begin(inode, page, pos, len, lseg,
+                                          (struct pnfs_fsdata **)

fsdata);

fs/nfs/pnfs.h pnfs_ld_layoutret_on_setattr(struct inode *inode)

Quoted 6 lines

fs/nfs/pnfs.h +6 -0

+       return status;
+}
+
+/* CAREFUL - what happens if copied < len??? */
+static inline int pnfs_write_end(struct file *filp, struct page *page,
+                                loff_t pos, unsigned len, unsigned

copied,

fs/nfs/pnfs.h pnfs_ld_layoutret_on_setattr(struct inode *inode)

Quoted 8 lines

fs/nfs/pnfs.h +8 -0

+                                struct pnfs_layout_segment *lseg)
+{
+       struct inode *inode = filp->f_dentry->d_inode;
+       struct nfs_server *nfss = NFS_SERVER(inode);
+
+       if (nfss->pnfs_curr_ld && nfss->pnfs_curr_ld->write_end)
+               return nfss->pnfs_curr_ld->write_end(inode, page, pos, len,
+                                                    copied,

lseg);

fs/nfs/pnfs.h pnfs_ld_layoutret_on_setattr(struct inode *inode)

Quoted 11 lines

fs/nfs/pnfs.h +11 -0

+       else
+               return 0;
+}
+
+static inline void pnfs_write_end_cleanup(struct file *filp, void *fsdata)
+{
+       struct nfs_server *nfss = NFS_SERVER(filp->f_dentry->d_inode);
+
+       if (fsdata && nfss->pnfs_curr_ld) {
+               if (nfss->pnfs_curr_ld->write_end_cleanup)
+                       nfss->pnfs_curr_ld->write_end_cleanup(filp,

fsdata);

fs/nfs/pnfs.h static inline void pnfs_pageio_init(struct

Quoted 9 lines

fs/nfs/pnfs.h +5 -0

+               if (nfss->pnfs_curr_ld->write_begin)
+                       pnfs_free_fsdata(fsdata);
+       }
+}
+
 static inline int pnfs_return_layout(struct inode *ino)
 {
       struct nfs_inode *nfsi = NFS_I(ino);

@@ -317,6 +386,19 @@ static inline void pnfs_pageio_init(struct

nfs_pageio_descriptor *pgio,

fs/nfs/pnfs.h static inline void pnfs_pageio_init(struct nfs_pageio_descriptor *pgio,

Quoted 8 lines

fs/nfs/pnfs.h +5 -0

               pgio->pg_test = ld->pg_test;
 }
 
+static inline struct pnfs_layout_segment *
+nfs4_pull_lseg_from_fsdata(struct file *filp, void *fsdata)
+{
+       if (fsdata) {
+               struct nfs_server *nfss =

NFS_SERVER(filp->f_dentry->d_inode);

fs/nfs/pnfs.h pnfs_update_layout(struct inode *ino, struct

Quoted 12 lines

fs/nfs/pnfs.h +8 -0

+
+               if (nfss->pnfs_curr_ld && nfss->pnfs_curr_ld->write_begin)
+                       return ((struct pnfs_fsdata *) fsdata)->lseg;
+               return (struct pnfs_layout_segment *)fsdata;
+       }
+       return NULL;
+}
+
 #else  /* CONFIG_NFS_V4_1 */
 
 static inline void pnfs_destroy_all_layouts(struct nfs_client *clp)

@@ -345,6 +427,12 @@ pnfs_update_layout(struct inode *ino, struct

nfs_open_context *ctx,

Quoted 17 lines

fs/nfs/pnfs.h +7 -0

       return NULL;
 }
 
+static inline int pnfs_grow_ok(struct pnfs_layout_segment *lseg,
+                              struct pnfs_fsdata *fsdata)
+{
+       return 1;
+}
+
 static inline enum pnfs_try_status
 pnfs_try_to_read_data(struct nfs_read_data *data,
                     const struct rpc_call_ops *call_ops)

@@ -364,6 +452,26 @@ static inline int pnfs_return_layout(struct inode *ino)
       return 0;
 }
 
+static inline int pnfs_write_begin(struct file *filp, struct page *page,

fs/nfs/pnfs.h static inline int pnfs_return_layout(struct inode *ino) · line 465

Quoted 10 lines

fs/nfs/pnfs.h +10 -0

+                                  loff_t pos, unsigned len,
+                                  struct pnfs_layout_segment *lseg,
+                                  void **fsdata)
+{
+       *fsdata = NULL;
+       return 0;
+}
+
+static inline int pnfs_write_end(struct file *filp, struct page *page,
+                                loff_t pos, unsigned len, unsigned

copied,

fs/nfs/pnfs.h static inline int pnfs_layoutcommit_inode(struct inode

Quoted 13 lines

fs/nfs/pnfs.h +9 -0

+                                struct pnfs_layout_segment *lseg)
+{
+       return 0;
+}
+
+static inline void pnfs_write_end_cleanup(struct file *filp, void *fsdata)
+{
+}
+
 static inline bool
 pnfs_ld_layoutret_on_setattr(struct inode *inode)
 {

@@ -435,6 +543,13 @@ static inline int pnfs_layoutcommit_inode(struct inode

*inode, bool sync)

fs/nfs/write.c static int nfs_writepage_setup(struct nfs_open_context

Quoted 29 lines

fs/nfs/pnfs.h +7 -0

 static inline void nfs4_deviceid_purge_client(struct nfs_client *ncl)
 {
 }
+
+static inline struct pnfs_layout_segment *
+nfs4_pull_lseg_from_fsdata(struct file *filp, void *fsdata)
+{
+       return NULL;
+}
+
 #endif /* CONFIG_NFS_V4_1 */
 
 #endif /* FS_NFS_PNFS_H */

fs/nfs/write.c +3 -1

index e268e3b..75e2a6b 100644
@@ -673,7 +673,9 @@ out:
 }
 
 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
-               unsigned int offset, unsigned int count)
+               unsigned int offset, unsigned int count,
+               struct pnfs_layout_segment *lseg, void *fsdata)
+
 {
       struct nfs_page *req;
 

@@ -681,7 +683,8 @@ static int nfs_writepage_setup(struct nfs_open_context

*ctx, struct page *page,

fs/nfs/write.c static int nfs_write_pageuptodate(struct page *page,

Quoted 10 lines

fs/nfs/write.c +2 -1

       if (IS_ERR(req))
               return PTR_ERR(req);
       /* Update file length */
-       nfs_grow_file(page, offset, count);
+       if (pnfs_grow_ok(lseg, fsdata))
+               nfs_grow_file(page, offset, count);
       nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
       nfs_mark_request_dirty(req);
       nfs_clear_page_tag_locked(req);

@@ -734,7 +737,8 @@ static int nfs_write_pageuptodate(struct page *page,

struct inode *inode)

Quoted 23 lines

fs/nfs/write.c +3 -2

 * things with a page scheduled for an RPC call (e.g. invalidate it).
 */
 int nfs_updatepage(struct file *file, struct page *page,
-               unsigned int offset, unsigned int count)
+               unsigned int offset, unsigned int count,
+               struct pnfs_layout_segment *lseg, void *fsdata)
 {
       struct nfs_open_context *ctx = nfs_file_open_context(file);
       struct inode    *inode = page->mapping->host;

@@ -759,7 +763,7 @@ int nfs_updatepage(struct file *file, struct page *page,
               offset = 0;
       }
 
-       status = nfs_writepage_setup(ctx, page, offset, count);
+       status = nfs_writepage_setup(ctx, page, offset, count, lseg, fsdata);
       if (status < 0)
               nfs_set_pageerror(page);
 

include/linux/nfs_fs.h +0 -0

index 1b93b9c..be1ac1d 100644

include/linux/nfs_fs.h extern int nfs_congestion_kb; · line 513

Quoted 4 lines

include/linux/nfs_fs.h +0 -1

@@ -510,7 +510,8 @@ extern int  nfs_congestion_kb;
 extern int  nfs_writepage(struct page *page, struct writeback_control *wbc);
 extern int  nfs_writepages(struct address_space *, struct writeback_control *);
 extern int  nfs_flush_incompatible(struct file *file, struct page *page);
-extern int  nfs_updatepage(struct file *, struct page *, unsigned int, unsigned

int);

include/linux/nfs_fs.h extern int nfs_congestion_kb; · line 513

Quoted 1 line

include/linux/nfs_fs.h +1 -0

+extern int  nfs_updatepage(struct file *, struct page *, unsigned int, unsigned

int,

Quoted 7 lines

include/linux/nfs_fs.h +1 -1

+                       struct pnfs_layout_segment *, void *);
 extern void nfs_writeback_done(struct rpc_task *, struct nfs_write_data *);
 
 /*
--

1.7.4.1

Quoted 5 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Hi, Fred,

Quoted 49 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:44 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 04/34] pnfs: hook nfs_write_begin/end to allow layout driver manipulation

Quoted 9 lines · Jim Rees

From: Peng Tao <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Reported-by Alexandros Batsakis <[email protected]>

Signed-off-by Andy Adamson <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

Quoted 4 lines

---

 fs/nfs/file.c          |   26 ++++++++++-
 fs/nfs/pnfs.c          |   41 +++++++++++++++++
 fs/nfs/pnfs.h          |  115

++++++++++++++++++++++++++++++++++++++++++++++++

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct

Quoted 9 lines

 fs/nfs/write.c         |   12 +++--
 include/linux/nfs_fs.h |    3 +-

 5 files changed, 189 insertions(+), 8 deletions(-)

fs/nfs/file.c +0 -0

index 2f093ed..1768762 100644
@@ -384,12 +384,15 @@ static int nfs_write_begin(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct address_space *mapping, · line 395

Quoted 13 lines

fs/nfs/file.c +4 -1

        pgoff_t index = pos >> PAGE_CACHE_SHIFT;
        struct page *page;
        int once_thru = 0;
+       struct pnfs_layout_segment *lseg;
 
        dfprintk(PAGECACHE, "NFS: write_begin(%s/%s(%ld), %u@%lld)\n",
                file->f_path.dentry->d_parent->d_name.name,
                file->f_path.dentry->d_name.name,
                mapping->host->i_ino, len, (long long) pos);
-
+       lseg = pnfs_update_layout(mapping->host,
+                                 nfs_file_open_context(file),
+                                 pos, len, IOMODE_RW, GFP_NOFS);

Quoted 7 lines

This looks like it is left over from before the rearrangements done to where pnfs_update_layout.
In particular, we don't want to hold the reference on the lseg from here until flush time.  And there
seems to be no reason to.  If the client needs a layout to deal with read-in here, it should instead
trigger the nfs_want_read_modify_write clause.

Yes, you are right. Directly calling pnfs_update_layout here can be avoided.
But it seems triggering nfs_want_read_modify_write will acquire a read-only layout segment via readpage code path.
For write, client will need a read-write layout segment so it would mean two layoutget for each new segment (one in nfs_readpage and one at flush time). It may not be good for performance. Does current generic code have method to avoid this?

Thanks,
Tao

Quoted 63 lines · On Tue, Jun 14, 2011 at 7:01 AM,  <[email protected]> wrote:

Hi, Fred,

Quoted 49 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:44 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 04/34] pnfs: hook nfs_write_begin/end to allow layout driver manipulation

Quoted 9 lines · Jim Rees

From: Peng Tao <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Reported-by Alexandros Batsakis <[email protected]>

Signed-off-by Andy Adamson <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

Quoted 4 lines

---

 fs/nfs/file.c          |   26 ++++++++++-
 fs/nfs/pnfs.c          |   41 +++++++++++++++++
 fs/nfs/pnfs.h          |  115

++++++++++++++++++++++++++++++++++++++++++++++++

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct

Quoted 9 lines

 fs/nfs/write.c         |   12 +++--
 include/linux/nfs_fs.h |    3 +-

 5 files changed, 189 insertions(+), 8 deletions(-)

fs/nfs/file.c +0 -0

index 2f093ed..1768762 100644
@@ -384,12 +384,15 @@ static int nfs_write_begin(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct address_space *mapping, · line 395

Quoted 13 lines

fs/nfs/file.c +4 -1

        pgoff_t index = pos >> PAGE_CACHE_SHIFT;
        struct page *page;
        int once_thru = 0;
+       struct pnfs_layout_segment *lseg;
 
        dfprintk(PAGECACHE, "NFS: write_begin(%s/%s(%ld), %u@%lld)\n",
                file->f_path.dentry->d_parent->d_name.name,
                file->f_path.dentry->d_name.name,
                mapping->host->i_ino, len, (long long) pos);
-
+       lseg = pnfs_update_layout(mapping->host,
+                                 nfs_file_open_context(file),
+                                 pos, len, IOMODE_RW, GFP_NOFS);

Quoted 7 lines

This looks like it is left over from before the rearrangements done to where pnfs_update_layout.
In particular, we don't want to hold the reference on the lseg from here until flush time.  And there
seems to be no reason to.  If the client needs a layout to deal with read-in here, it should instead
trigger the nfs_want_read_modify_write clause.

Yes, you are right. Directly calling pnfs_update_layout here can be avoided.
But it seems triggering nfs_want_read_modify_write will acquire a read-only layout segment via readpage code path.
For write, client will need a read-write layout segment so it would mean two layoutget for each new segment (one in nfs_readpage and one at flush time). It may not be good for performance. Does current generic code have method to avoid this?

Quoted 3 lines

Thanks,
Tao

No.  However, note that this only hits in the case where you are doing subpage writes.

Fred
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 64 lines · Fred Isaman

Quoted 63 lines · On Tue, Jun 14, 2011 at 7:01 AM,  <[email protected]> wrote:

Hi, Fred,

Quoted 49 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Fred Isaman
Sent: Monday, June 13, 2011 10:44 PM
To: Jim Rees
Cc: [email protected]; peter honeyman
Subject: Re: [PATCH 04/34] pnfs: hook nfs_write_begin/end to allow layout driver manipulation

Quoted 9 lines · Jim Rees

From: Peng Tao <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Reported-by Alexandros Batsakis <[email protected]>

Signed-off-by Andy Adamson <[email protected]>

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

Quoted 4 lines

---

 fs/nfs/file.c          |   26 ++++++++++-
 fs/nfs/pnfs.c          |   41 +++++++++++++++++
 fs/nfs/pnfs.h          |  115

++++++++++++++++++++++++++++++++++++++++++++++++

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct

Quoted 9 lines

 fs/nfs/write.c         |   12 +++--
 include/linux/nfs_fs.h |    3 +-

 5 files changed, 189 insertions(+), 8 deletions(-)

fs/nfs/file.c +0 -0

index 2f093ed..1768762 100644
@@ -384,12 +384,15 @@ static int nfs_write_begin(struct file *file, struct

address_space *mapping,

fs/nfs/file.c static int nfs_write_begin(struct file *file, struct address_space *mapping, · line 395

Quoted 13 lines

fs/nfs/file.c +4 -1

        pgoff_t index = pos >> PAGE_CACHE_SHIFT;
        struct page *page;
        int once_thru = 0;
+       struct pnfs_layout_segment *lseg;
 
        dfprintk(PAGECACHE, "NFS: write_begin(%s/%s(%ld), %u@%lld)\n",
                file->f_path.dentry->d_parent->d_name.name,
                file->f_path.dentry->d_name.name,
                mapping->host->i_ino, len, (long long) pos);
-
+       lseg = pnfs_update_layout(mapping->host,
+                                 nfs_file_open_context(file),
+                                 pos, len, IOMODE_RW, GFP_NOFS);

Quoted 7 lines

This looks like it is left over from before the rearrangements done to where pnfs_update_layout.
In particular, we don't want to hold the reference on the lseg from here until flush time.  And there
seems to be no reason to.  If the client needs a layout to deal with read-in here, it should instead
trigger the nfs_want_read_modify_write clause.

Yes, you are right. Directly calling pnfs_update_layout here can be avoided.
But it seems triggering nfs_want_read_modify_write will acquire a read-only layout segment via readpage code path.
For write, client will need a read-write layout segment so it would mean two layoutget for each new segment (one in nfs_readpage and one at flush time). It may not be good for performance. Does current generic code have method to avoid this?

Quoted 6 lines

Quoted 3 lines

Thanks,
Tao

No.  However, note that this only hits in the case where you are doing subpage writes.

block layout driver need the segment to determine if it should dirty other pages in the same fsblock based on if it is a first write to an INVALID extent. So it still hits whenever an fsblock is dirtied for the first time.

Thanks,
Tao

From: Peng Tao <[email protected]>

Block layout needs it to determine IO size.

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Tao Guo <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/client.c           |    1 +
 fs/nfs/nfs4_fs.h          |    2 +-
 fs/nfs/nfs4proc.c         |    5 +-
 fs/nfs/nfs4xdr.c          |  101 +++++++++++++++++++++++++++++++++++++--------
 include/linux/nfs_fs_sb.h |    4 +-
 include/linux/nfs_xdr.h   |    3 +-
 6 files changed, 93 insertions(+), 23 deletions(-)

fs/nfs/client.c +1 -0

index 6bdb7da0..b2c6920 100644
@@ -937,6 +937,7 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, struct nfs_fh *mntf
 	if (server->wsize > NFS_MAX_FILE_IO_SIZE)
 		server->wsize = NFS_MAX_FILE_IO_SIZE;
 	server->wpages = (server->wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+	server->pnfs_blksize = fsinfo->blksize;
 	set_pnfs_layoutdriver(server, mntfh, fsinfo->layouttype);
 
 	server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL);

fs/nfs/nfs4_fs.h +1 -1

index c4a6983..5725a7e 100644
@@ -315,7 +315,7 @@ extern const struct nfs4_minor_version_ops *nfs_v4_minor_ops[];
 extern const u32 nfs4_fattr_bitmap[2];
 extern const u32 nfs4_statfs_bitmap[2];
 extern const u32 nfs4_pathconf_bitmap[2];
-extern const u32 nfs4_fsinfo_bitmap[2];
+extern const u32 nfs4_fsinfo_bitmap[3];
 extern const u32 nfs4_fs_locations_bitmap[2];
 
 /* nfs4renewd.c */

fs/nfs/nfs4proc.c +3 -2

index 4a5ad93..5246db8 100644
@@ -137,12 +137,13 @@ const u32 nfs4_pathconf_bitmap[2] = {
 	0
 };
 
-const u32 nfs4_fsinfo_bitmap[2] = { FATTR4_WORD0_MAXFILESIZE
+const u32 nfs4_fsinfo_bitmap[3] = { FATTR4_WORD0_MAXFILESIZE
 			| FATTR4_WORD0_MAXREAD
 			| FATTR4_WORD0_MAXWRITE
 			| FATTR4_WORD0_LEASE_TIME,
 			FATTR4_WORD1_TIME_DELTA
-			| FATTR4_WORD1_FS_LAYOUT_TYPES
+			| FATTR4_WORD1_FS_LAYOUT_TYPES,
+			FATTR4_WORD2_LAYOUT_BLKSIZE
 };
 
 const u32 nfs4_fs_locations_bitmap[2] = {

fs/nfs/nfs4xdr.c +83 -18

index 3620c45..fdcbd8f 100644
@@ -91,7 +91,7 @@ static int nfs4_stat_to_errno(int);
 #define encode_getfh_maxsz      (op_encode_hdr_maxsz)
 #define decode_getfh_maxsz      (op_decode_hdr_maxsz + 1 + \
 				((3+NFS4_FHSIZE) >> 2))
-#define nfs4_fattr_bitmap_maxsz 3
+#define nfs4_fattr_bitmap_maxsz 4
 #define encode_getattr_maxsz    (op_encode_hdr_maxsz + nfs4_fattr_bitmap_maxsz)
 #define nfs4_name_maxsz		(1 + ((3 + NFS4_MAXNAMLEN) >> 2))
 #define nfs4_path_maxsz		(1 + ((3 + NFS4_MAXPATHLEN) >> 2))

@@ -113,7 +113,11 @@ static int nfs4_stat_to_errno(int);
 #define encode_restorefh_maxsz  (op_encode_hdr_maxsz)
 #define decode_restorefh_maxsz  (op_decode_hdr_maxsz)
 #define encode_fsinfo_maxsz	(encode_getattr_maxsz)
-#define decode_fsinfo_maxsz	(op_decode_hdr_maxsz + 15)
+/* The 5 accounts for the PNFS attributes, and assumes that at most three
+ * layout types will be returned.
+ */
+#define decode_fsinfo_maxsz	(op_decode_hdr_maxsz + \
+				 nfs4_fattr_bitmap_maxsz + 4 + 8 + 5)
 #define encode_renew_maxsz	(op_encode_hdr_maxsz + 3)
 #define decode_renew_maxsz	(op_decode_hdr_maxsz)
 #define encode_setclientid_maxsz \

@@ -1095,6 +1099,35 @@ static void encode_getattr_two(struct xdr_stream *xdr, uint32_t bm0, uint32_t bm
 	hdr->replen += decode_getattr_maxsz;
 }
 
+static void
+encode_getattr_three(struct xdr_stream *xdr,
+		     uint32_t bm0, uint32_t bm1, uint32_t bm2,
+		     struct compound_hdr *hdr)
+{
+	__be32 *p;
+
+	p = reserve_space(xdr, 4);
+	*p = cpu_to_be32(OP_GETATTR);
+	if (bm2) {
+		p = reserve_space(xdr, 16);
+		*p++ = cpu_to_be32(3);
+		*p++ = cpu_to_be32(bm0);
+		*p++ = cpu_to_be32(bm1);
+		*p = cpu_to_be32(bm2);
+	} else if (bm1) {
+		p = reserve_space(xdr, 12);
+		*p++ = cpu_to_be32(2);
+		*p++ = cpu_to_be32(bm0);
+		*p = cpu_to_be32(bm1);
+	} else {
+		p = reserve_space(xdr, 8);
+		*p++ = cpu_to_be32(1);
+		*p = cpu_to_be32(bm0);
+	}
+	hdr->nops++;
+	hdr->replen += decode_getattr_maxsz;
+}
+
 static void encode_getfattr(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)
 {
 	encode_getattr_two(xdr, bitmask[0] & nfs4_fattr_bitmap[0],

@@ -1103,8 +1136,11 @@ static void encode_getfattr(struct xdr_stream *xdr, const u32* bitmask, struct c
 
 static void encode_fsinfo(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)
 {
-	encode_getattr_two(xdr, bitmask[0] & nfs4_fsinfo_bitmap[0],
-			   bitmask[1] & nfs4_fsinfo_bitmap[1], hdr);
+	encode_getattr_three(xdr,
+			     bitmask[0] & nfs4_fsinfo_bitmap[0],
+			     bitmask[1] & nfs4_fsinfo_bitmap[1],
+			     bitmask[2] & nfs4_fsinfo_bitmap[2],
+			     hdr);
 }
 
 static void encode_fs_locations(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)

@@ -2575,7 +2611,7 @@ static void nfs4_xdr_enc_setclientid_confirm(struct rpc_rqst *req,
 	struct compound_hdr hdr = {
 		.nops	= 0,
 	};
-	const u32 lease_bitmap[2] = { FATTR4_WORD0_LEASE_TIME, 0 };
+	const u32 lease_bitmap[3] = { FATTR4_WORD0_LEASE_TIME, 0, 0 };
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_setclientid_confirm(xdr, arg, &hdr);

@@ -2719,7 +2755,7 @@ static void nfs4_xdr_enc_get_lease_time(struct rpc_rqst *req,
 	struct compound_hdr hdr = {
 		.minorversion = nfs4_xdr_minorversion(&args->la_seq_args),
 	};
-	const u32 lease_bitmap[2] = { FATTR4_WORD0_LEASE_TIME, 0 };
+	const u32 lease_bitmap[3] = { FATTR4_WORD0_LEASE_TIME, 0, 0 };
 
 	encode_compound_hdr(xdr, req, &hdr);
 	encode_sequence(xdr, &args->la_seq_args, &hdr);

@@ -2947,14 +2983,17 @@ static int decode_attr_bitmap(struct xdr_stream *xdr, uint32_t *bitmap)
 		goto out_overflow;
 	bmlen = be32_to_cpup(p);
 
-	bitmap[0] = bitmap[1] = 0;
+	bitmap[0] = bitmap[1] = bitmap[2] = 0;
 	p = xdr_inline_decode(xdr, (bmlen << 2));
 	if (unlikely(!p))
 		goto out_overflow;
 	if (bmlen > 0) {
 		bitmap[0] = be32_to_cpup(p++);
-		if (bmlen > 1)
-			bitmap[1] = be32_to_cpup(p);
+		if (bmlen > 1) {
+			bitmap[1] = be32_to_cpup(p++);
+			if (bmlen > 2)
+				bitmap[2] = be32_to_cpup(p);
+		}
 	}
 	return 0;
 out_overflow:

@@ -2986,8 +3025,9 @@ static int decode_attr_supported(struct xdr_stream *xdr, uint32_t *bitmap, uint3
 			return ret;
 		bitmap[0] &= ~FATTR4_WORD0_SUPPORTED_ATTRS;
 	} else
-		bitmask[0] = bitmask[1] = 0;
-	dprintk("%s: bitmask=%08x:%08x\n", __func__, bitmask[0], bitmask[1]);
+		bitmask[0] = bitmask[1] = bitmask[2] = 0;
+	dprintk("%s: bitmask=%08x:%08x:%08x\n", __func__,
+		bitmask[0], bitmask[1], bitmask[2]);
 	return 0;
 }
 

@@ -4041,7 +4081,7 @@ out_overflow:
 static int decode_server_caps(struct xdr_stream *xdr, struct nfs4_server_caps_res *res)
 {
 	__be32 *savep;
-	uint32_t attrlen, bitmap[2] = {0};
+	uint32_t attrlen, bitmap[3] = {0};
 	int status;
 
 	if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4067,7 +4107,7 @@ xdr_error:
 static int decode_statfs(struct xdr_stream *xdr, struct nfs_fsstat *fsstat)
 {
 	__be32 *savep;
-	uint32_t attrlen, bitmap[2] = {0};
+	uint32_t attrlen, bitmap[3] = {0};
 	int status;
 
 	if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4099,7 +4139,7 @@ xdr_error:
 static int decode_pathconf(struct xdr_stream *xdr, struct nfs_pathconf *pathconf)
 {
 	__be32 *savep;
-	uint32_t attrlen, bitmap[2] = {0};
+	uint32_t attrlen, bitmap[3] = {0};
 	int status;
 
 	if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4239,7 +4279,7 @@ static int decode_getfattr_generic(struct xdr_stream *xdr, struct nfs_fattr *fat
 {
 	__be32 *savep;
 	uint32_t attrlen,
-		 bitmap[2] = {0};
+		 bitmap[3] = {0};
 	int status;
 
 	status = decode_op_hdr(xdr, OP_GETATTR);

@@ -4325,10 +4365,32 @@ static int decode_attr_pnfstype(struct xdr_stream *xdr, uint32_t *bitmap,
 	return status;
 }
 
+/*
+ * The prefered block size for layout directed io
+ */
+static int decode_attr_layout_blksize(struct xdr_stream *xdr, uint32_t *bitmap,
+				      uint32_t *res)
+{
+	__be32 *p;
+
+	dprintk("%s: bitmap is %x\n", __func__, bitmap[2]);
+	*res = 0;
+	if (bitmap[2] & FATTR4_WORD2_LAYOUT_BLKSIZE) {
+		p = xdr_inline_decode(xdr, 4);
+		if (unlikely(!p)) {
+			print_overflow_msg(__func__, xdr);
+			return -EIO;
+		}
+		*res = be32_to_cpup(p);
+		bitmap[2] &= ~FATTR4_WORD2_LAYOUT_BLKSIZE;
+	}
+	return 0;
+}
+
 static int decode_fsinfo(struct xdr_stream *xdr, struct nfs_fsinfo *fsinfo)
 {
 	__be32 *savep;
-	uint32_t attrlen, bitmap[2];
+	uint32_t attrlen, bitmap[3];
 	int status;
 
 	if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4356,6 +4418,9 @@ static int decode_fsinfo(struct xdr_stream *xdr, struct nfs_fsinfo *fsinfo)
 	status = decode_attr_pnfstype(xdr, bitmap, &fsinfo->layouttype);
 	if (status != 0)
 		goto xdr_error;
+	status = decode_attr_layout_blksize(xdr, bitmap, &fsinfo->blksize);
+	if (status)
+		goto xdr_error;
 
 	status = verify_attr_len(xdr, savep, attrlen);
 xdr_error:

@@ -4775,7 +4840,7 @@ static int decode_getacl(struct xdr_stream *xdr, struct rpc_rqst *req,
 {
 	__be32 *savep;
 	uint32_t attrlen,
-		 bitmap[2] = {0};
+		 bitmap[3] = {0};
 	struct kvec *iov = req->rq_rcv_buf.head;
 	int status;
 

@@ -6605,7 +6670,7 @@ out:
 int nfs4_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
 		       int plus)
 {
-	uint32_t bitmap[2] = {0};
+	uint32_t bitmap[3] = {0};
 	uint32_t len;
 	__be32 *p = xdr_inline_decode(xdr, 4);
 	if (unlikely(!p))

include/linux/nfs_fs_sb.h +3 -1

index 87694ca..79cc4ca 100644
@@ -130,7 +130,7 @@ struct nfs_server {
 #endif
 
 #ifdef CONFIG_NFS_V4
-	u32			attr_bitmask[2];/* V4 bitmask representing the set
+	u32			attr_bitmask[3];/* V4 bitmask representing the set
 						   of attributes supported on this
 						   filesystem */
 	u32			cache_consistency_bitmask[2];

@@ -143,6 +143,8 @@ struct nfs_server {
 						   filesystem */
 	struct pnfs_layoutdriver_type  *pnfs_curr_ld; /* Active layout driver */
 	struct rpc_wait_queue	roc_rpcwaitq;
+	void			*pnfs_ld_data; /* per mount point data */
+	u32			pnfs_blksize; /* layout_blksize attr */
 
 	/* the following fields are protected by nfs_client->cl_lock */
 	struct rb_root		state_owners;

include/linux/nfs_xdr.h +2 -1

index 00442f5..a9c43ba 100644
@@ -122,6 +122,7 @@ struct nfs_fsinfo {
 	struct timespec		time_delta; /* server time granularity */
 	__u32			lease_time; /* in seconds */
 	__u32			layouttype; /* supported pnfs layout driver */
+	__u32			blksize; /* preferred pnfs io block size */
 };
 
 struct nfs_fsstat {

@@ -954,7 +955,7 @@ struct nfs4_server_caps_arg {
 };
 
 struct nfs4_server_caps_res {
-	u32				attr_bitmask[2];
+	u32				attr_bitmask[3];
 	u32				acl_bitmask;
 	u32				has_links;
 	u32				has_symlinks;

Quoted 1 line · Jim Rees

From: Peng Tao <[email protected]>

Jim, please revise the authorshop of the different patches. This for example, was originally authored by
Fred Isaman <[email protected]> (see commit 5c5a76f)

Benny

Quoted 321 lines

Block layout needs it to determine IO size.

Signed-off-by Fred Isaman <[email protected]>

Signed-off-by Tao Guo <[email protected]>

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/client.c           |    1 +
 fs/nfs/nfs4_fs.h          |    2 +-
 fs/nfs/nfs4proc.c         |    5 +-
 fs/nfs/nfs4xdr.c          |  101 +++++++++++++++++++++++++++++++++++++-------- include/linux/nfs_fs_sb.h |    4 +-
 include/linux/nfs_xdr.h   |    3 +-

 6 files changed, 93 insertions(+), 23 deletions(-)

fs/nfs/client.c +1 -0

index 6bdb7da0..b2c6920 100644
@@ -937,6 +937,7 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, struct nfs_fh *mntf
       if (server->wsize > NFS_MAX_FILE_IO_SIZE)
               server->wsize = NFS_MAX_FILE_IO_SIZE;
       server->wpages = (server->wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+     server->pnfs_blksize = fsinfo->blksize;
       set_pnfs_layoutdriver(server, mntfh, fsinfo->layouttype);
 
       server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL);

fs/nfs/nfs4_fs.h +1 -1

index c4a6983..5725a7e 100644
@@ -315,7 +315,7 @@ extern const struct nfs4_minor_version_ops *nfs_v4_minor_ops[];
  extern const u32 nfs4_fattr_bitmap[2];
  extern const u32 nfs4_statfs_bitmap[2];
  extern const u32 nfs4_pathconf_bitmap[2];
-extern const u32 nfs4_fsinfo_bitmap[2];
+extern const u32 nfs4_fsinfo_bitmap[3];
  extern const u32 nfs4_fs_locations_bitmap[2];
 
  /* nfs4renewd.c */

fs/nfs/nfs4proc.c +0 -0

index 4a5ad93..5246db8 100644
@@ -137,12 +137,13 @@ const u32 nfs4_pathconf_bitmap[2] = {

      0
 };

fs/nfs/nfs4proc.c +1 -1

-const u32 nfs4_fsinfo_bitmap[2] = { FATTR4_WORD0_MAXFILESIZE
+const u32 nfs4_fsinfo_bitmap[3] = { FATTR4_WORD0_MAXFILESIZE

                      | FATTR4_WORD0_MAXREAD
                      | FATTR4_WORD0_MAXWRITE
                      | FATTR4_WORD0_LEASE_TIME,

                      FATTR4_WORD1_TIME_DELTA
-                     | FATTR4_WORD1_FS_LAYOUT_TYPES
+                     | FATTR4_WORD1_FS_LAYOUT_TYPES,
+                     FATTR4_WORD2_LAYOUT_BLKSIZE };

 const u32 nfs4_fs_locations_bitmap[2] = {

fs/nfs/nfs4xdr.c +48 -9

index 3620c45..fdcbd8f 100644
@@ -91,7 +91,7 @@ static int nfs4_stat_to_errno(int);
  #define encode_getfh_maxsz      (op_encode_hdr_maxsz)
  #define decode_getfh_maxsz      (op_decode_hdr_maxsz + 1 + \
                               ((3+NFS4_FHSIZE) >> 2))
-#define nfs4_fattr_bitmap_maxsz 3
+#define nfs4_fattr_bitmap_maxsz 4
  #define encode_getattr_maxsz    (op_encode_hdr_maxsz + nfs4_fattr_bitmap_maxsz)
  #define nfs4_name_maxsz              (1 + ((3 + NFS4_MAXNAMLEN) >> 2))
  #define nfs4_path_maxsz              (1 + ((3 + NFS4_MAXPATHLEN) >> 2))

@@ -113,7 +113,11 @@ static int nfs4_stat_to_errno(int);
  #define encode_restorefh_maxsz  (op_encode_hdr_maxsz)
  #define decode_restorefh_maxsz  (op_decode_hdr_maxsz)
  #define encode_fsinfo_maxsz  (encode_getattr_maxsz)
-#define decode_fsinfo_maxsz  (op_decode_hdr_maxsz + 15)
+/* The 5 accounts for the PNFS attributes, and assumes that at most three
+ * layout types will be returned.
+ */
+#define decode_fsinfo_maxsz  (op_decode_hdr_maxsz + \
+                              nfs4_fattr_bitmap_maxsz + 4 + 8 + 5)
  #define encode_renew_maxsz   (op_encode_hdr_maxsz + 3)
  #define decode_renew_maxsz   (op_decode_hdr_maxsz)
  #define encode_setclientid_maxsz \

@@ -1095,6 +1099,35 @@ static void encode_getattr_two(struct xdr_stream *xdr, uint32_t bm0, uint32_t bm
       hdr->replen += decode_getattr_maxsz;
  }
 
+static void
+encode_getattr_three(struct xdr_stream *xdr,
+                  uint32_t bm0, uint32_t bm1, uint32_t bm2,
+                  struct compound_hdr *hdr)
+{
+     __be32 *p;
+
+     p = reserve_space(xdr, 4);
+     *p = cpu_to_be32(OP_GETATTR);
+     if (bm2) {
+             p = reserve_space(xdr, 16);
+             *p++ = cpu_to_be32(3);
+             *p++ = cpu_to_be32(bm0);
+             *p++ = cpu_to_be32(bm1);
+             *p = cpu_to_be32(bm2);
+     } else if (bm1) {
+             p = reserve_space(xdr, 12);
+             *p++ = cpu_to_be32(2);
+             *p++ = cpu_to_be32(bm0);
+             *p = cpu_to_be32(bm1);
+     } else {
+             p = reserve_space(xdr, 8);
+             *p++ = cpu_to_be32(1);
+             *p = cpu_to_be32(bm0);
+     }
+     hdr->nops++;
+     hdr->replen += decode_getattr_maxsz;
+}
+
  static void encode_getfattr(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)
  {
       encode_getattr_two(xdr, bitmask[0] & nfs4_fattr_bitmap[0],

@@ -1103,8 +1136,11 @@ static void encode_getfattr(struct xdr_stream *xdr, const u32* bitmask, struct c
 
  static void encode_fsinfo(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)
  {
-     encode_getattr_two(xdr, bitmask[0] & nfs4_fsinfo_bitmap[0],
-                        bitmask[1] & nfs4_fsinfo_bitmap[1], hdr);
+     encode_getattr_three(xdr,
+                          bitmask[0] & nfs4_fsinfo_bitmap[0],
+                          bitmask[1] & nfs4_fsinfo_bitmap[1],
+                          bitmask[2] & nfs4_fsinfo_bitmap[2],
+                          hdr);
  }
 
  static void encode_fs_locations(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr)

@@ -2575,7 +2611,7 @@ static void nfs4_xdr_enc_setclientid_confirm(struct rpc_rqst *req,
       struct compound_hdr hdr = {
               .nops   = 0,
       };
-     const u32 lease_bitmap[2] = { FATTR4_WORD0_LEASE_TIME, 0 };
+     const u32 lease_bitmap[3] = { FATTR4_WORD0_LEASE_TIME, 0, 0 };
 
       encode_compound_hdr(xdr, req, &hdr);
       encode_setclientid_confirm(xdr, arg, &hdr);

@@ -2719,7 +2755,7 @@ static void nfs4_xdr_enc_get_lease_time(struct rpc_rqst *req,
       struct compound_hdr hdr = {
               .minorversion = nfs4_xdr_minorversion(&args->la_seq_args),
       };
-     const u32 lease_bitmap[2] = { FATTR4_WORD0_LEASE_TIME, 0 };
+     const u32 lease_bitmap[3] = { FATTR4_WORD0_LEASE_TIME, 0, 0 };
 
       encode_compound_hdr(xdr, req, &hdr);
       encode_sequence(xdr, &args->la_seq_args, &hdr);

@@ -2947,14 +2983,17 @@ static int decode_attr_bitmap(struct xdr_stream *xdr, uint32_t *bitmap)
               goto out_overflow;
       bmlen = be32_to_cpup(p);
 
-     bitmap[0] = bitmap[1] = 0;
+     bitmap[0] = bitmap[1] = bitmap[2] = 0;
       p = xdr_inline_decode(xdr, (bmlen << 2));
       if (unlikely(!p))
               goto out_overflow;
       if (bmlen > 0) {
               bitmap[0] = be32_to_cpup(p++);
-             if (bmlen > 1)
-                     bitmap[1] = be32_to_cpup(p);
+             if (bmlen > 1) {
+                     bitmap[1] = be32_to_cpup(p++);
+                     if (bmlen > 2)
+                             bitmap[2] = be32_to_cpup(p);
+             }
       }
       return 0;

 out_overflow:

fs/nfs/nfs4xdr.c +33 -7

@@ -2986,8 +3025,9 @@ static int decode_attr_supported(struct xdr_stream *xdr, uint32_t *bitmap, uint3
                       return ret;
               bitmap[0] &= ~FATTR4_WORD0_SUPPORTED_ATTRS;
       } else
-             bitmask[0] = bitmask[1] = 0;
-     dprintk("%s: bitmask=%08x:%08x\n", __func__, bitmask[0], bitmask[1]);
+             bitmask[0] = bitmask[1] = bitmask[2] = 0;
+     dprintk("%s: bitmask=%08x:%08x:%08x\n", __func__,
+             bitmask[0], bitmask[1], bitmask[2]);
       return 0;
  }
 

@@ -4041,7 +4081,7 @@ out_overflow:
  static int decode_server_caps(struct xdr_stream *xdr, struct nfs4_server_caps_res *res)
  {
       __be32 *savep;
-     uint32_t attrlen, bitmap[2] = {0};
+     uint32_t attrlen, bitmap[3] = {0};
       int status;
 
       if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4067,7 +4107,7 @@ xdr_error:
  static int decode_statfs(struct xdr_stream *xdr, struct nfs_fsstat *fsstat)
  {
       __be32 *savep;
-     uint32_t attrlen, bitmap[2] = {0};
+     uint32_t attrlen, bitmap[3] = {0};
       int status;
 
       if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4099,7 +4139,7 @@ xdr_error:
  static int decode_pathconf(struct xdr_stream *xdr, struct nfs_pathconf *pathconf)
  {
       __be32 *savep;
-     uint32_t attrlen, bitmap[2] = {0};
+     uint32_t attrlen, bitmap[3] = {0};
       int status;
 
       if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4239,7 +4279,7 @@ static int decode_getfattr_generic(struct xdr_stream *xdr, struct nfs_fattr *fat
  {
       __be32 *savep;
       uint32_t attrlen,
-              bitmap[2] = {0};
+              bitmap[3] = {0};
       int status;
 
       status = decode_op_hdr(xdr, OP_GETATTR);

@@ -4325,10 +4365,32 @@ static int decode_attr_pnfstype(struct xdr_stream *xdr, uint32_t *bitmap,
       return status;
  }
 
+/*
+ * The prefered block size for layout directed io
+ */
+static int decode_attr_layout_blksize(struct xdr_stream *xdr, uint32_t *bitmap,
+                                   uint32_t *res)
+{
+     __be32 *p;
+
+     dprintk("%s: bitmap is %x\n", __func__, bitmap[2]);
+     *res = 0;
+     if (bitmap[2] & FATTR4_WORD2_LAYOUT_BLKSIZE) {
+             p = xdr_inline_decode(xdr, 4);
+             if (unlikely(!p)) {
+                     print_overflow_msg(__func__, xdr);
+                     return -EIO;
+             }
+             *res = be32_to_cpup(p);
+             bitmap[2] &= ~FATTR4_WORD2_LAYOUT_BLKSIZE;
+     }
+     return 0;
+}
+
  static int decode_fsinfo(struct xdr_stream *xdr, struct nfs_fsinfo *fsinfo)
  {
       __be32 *savep;
-     uint32_t attrlen, bitmap[2];
+     uint32_t attrlen, bitmap[3];
       int status;
 
       if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0)

@@ -4356,6 +4418,9 @@ static int decode_fsinfo(struct xdr_stream *xdr, struct nfs_fsinfo *fsinfo)
       status = decode_attr_pnfstype(xdr, bitmap, &fsinfo->layouttype);
       if (status != 0)
               goto xdr_error;
+     status = decode_attr_layout_blksize(xdr, bitmap, &fsinfo->blksize);
+     if (status)
+             goto xdr_error;
 
       status = verify_attr_len(xdr, savep, attrlen);

 xdr_error:

fs/nfs/nfs4xdr.c +2 -2

@@ -4775,7 +4840,7 @@ static int decode_getacl(struct xdr_stream *xdr, struct rpc_rqst *req,
  {
       __be32 *savep;
       uint32_t attrlen,
-              bitmap[2] = {0};
+              bitmap[3] = {0};
       struct kvec *iov = req->rq_rcv_buf.head;
       int status;
 

@@ -6605,7 +6670,7 @@ out:
  int nfs4_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
                      int plus)
  {
-     uint32_t bitmap[2] = {0};
+     uint32_t bitmap[3] = {0};
       uint32_t len;
       __be32 *p = xdr_inline_decode(xdr, 4);
       if (unlikely(!p))

include/linux/nfs_fs_sb.h +1 -1

index 87694ca..79cc4ca 100644
@@ -130,7 +130,7 @@ struct nfs_server {
  #endif
 
  #ifdef CONFIG_NFS_V4
-     u32                     attr_bitmask[2];/* V4 bitmask representing the set
+     u32                     attr_bitmask[3];/* V4 bitmask representing the set

                                                 of attributes supported on this filesystem */
      u32                     cache_consistency_bitmask[2];

include/linux/nfs_fs_sb.h +0 -0

@@ -143,6 +143,8 @@ struct nfs_server {

                                                 filesystem */ struct pnfs_layoutdriver_type  *pnfs_curr_ld; /* Active layout driver */ struct rpc_wait_queue   roc_rpcwaitq;
+     void                    *pnfs_ld_data; /* per mount point data */
+     u32                     pnfs_blksize; /* layout_blksize attr */

      /* the following fields are protected by nfs_client->cl_lock */ struct rb_root          state_owners;

include/linux/nfs_xdr.h +0 -0

index 00442f5..a9c43ba 100644
@@ -122,6 +122,7 @@ struct nfs_fsinfo {

      struct timespec         time_delta; /* server time granularity */ __u32                   lease_time; /* in seconds */
      __u32                   layouttype; /* supported pnfs layout driver */
+     __u32                   blksize; /* preferred pnfs io block size */ };

 struct nfs_fsstat {

include/linux/nfs_xdr.h struct nfs4_server_caps_arg { · line 961

Quoted 8 lines

include/linux/nfs_xdr.h +1 -1

@@ -954,7 +955,7 @@ struct nfs4_server_caps_arg {
  };
 
  struct nfs4_server_caps_res {
-     u32                             attr_bitmask[2];
+     u32                             attr_bitmask[3];
       u32                             acl_bitmask;
       u32                             has_links;
       u32                             has_symlinks;

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

From: Peng Tao <[email protected]>

This gives layout driver a chance to cleanup structures they put in. Also ensure layoutcommit does not commit more than isize, as block layout driver may dirty pages beyond EOF.

Signed-off-by Andy Adamson <[email protected]>

[fixup layout header pointer for layoutcommit]

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/nfs4proc.c       |    1 +
 fs/nfs/nfs4xdr.c        |    3 ++-
 fs/nfs/pnfs.c           |   15 +++++++++++++++
 fs/nfs/pnfs.h           |    4 ++++
 include/linux/nfs_xdr.h |    1 +
 5 files changed, 23 insertions(+), 1 deletions(-)

fs/nfs/nfs4proc.c +0 -0

index 5246db8..e27a648 100644

fs/nfs/nfs4proc.c +0 -0

 	/* Matched by references in pnfs_set_layoutcommit */
 	put_lseg(data->lseg);
 	put_rpccred(data->cred);

fs/nfs/nfs4xdr.c +2 -1

index fdcbd8f..57295d1 100644
@@ -1963,7 +1963,7 @@ encode_layoutcommit(struct xdr_stream *xdr,
 	*p++ = cpu_to_be32(OP_LAYOUTCOMMIT);
 	/* Only whole file layouts */
 	p = xdr_encode_hyper(p, 0); /* offset */
-	p = xdr_encode_hyper(p, NFS4_MAX_UINT64); /* length */
+	p = xdr_encode_hyper(p, args->lastbytewritten+1); /* length */
 	*p++ = cpu_to_be32(0); /* reclaim */
 	p = xdr_encode_opaque_fixed(p, args->stateid.data, NFS4_STATEID_SIZE);
 	*p++ = cpu_to_be32(1); /* newoffset = TRUE */

@@ -5467,6 +5467,7 @@ static int decode_layoutcommit(struct xdr_stream *xdr,
 	int status;
 
 	status = decode_op_hdr(xdr, OP_LAYOUTCOMMIT);
+	res->status = status;
 	if (status)
 		return status;
 

fs/nfs/pnfs.c +15 -0

index e693718..48a06a1 100644
@@ -1248,6 +1248,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 {
 	struct nfs_inode *nfsi = NFS_I(wdata->inode);
 	loff_t end_pos = wdata->mds_offset + wdata->res.count;
+	loff_t isize = i_size_read(wdata->inode);
 	bool mark_as_dirty = false;
 
 	spin_lock(&nfsi->vfs_inode.i_lock);

@@ -1261,9 +1262,13 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 		dprintk("%s: Set layoutcommit for inode %lu ",
 			__func__, wdata->inode->i_ino);
 	}
+	if (end_pos > isize)
+		end_pos = isize;
 	if (end_pos > wdata->lseg->pls_end_pos)
 		wdata->lseg->pls_end_pos = end_pos;
 	spin_unlock(&nfsi->vfs_inode.i_lock);
+	dprintk("%s: lseg %p end_pos %llu\n",
+		__func__, wdata->lseg, wdata->lseg->pls_end_pos);
 
 	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
 	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */

@@ -1272,6 +1277,16 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 }
 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_cleanup_layoutcommit(struct inode *inode,
+                               struct nfs4_layoutcommit_data *data)
+{
+        struct nfs_server *nfss = NFS_SERVER(inode);
+
+        if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
+                nfss->pnfs_curr_ld->cleanup_layoutcommit(
+                                        NFS_I(inode)->layout, data);
+}
+
 void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
 {
 	/* lseg refcounting handled directly in nfs_write_end */

fs/nfs/pnfs.h +4 -0

index 525ec55..5048898 100644
@@ -127,6 +127,9 @@ struct pnfs_layoutdriver_type {
 				     struct xdr_stream *xdr,
 				     const struct nfs4_layoutreturn_args *args);
 
+        void (*cleanup_layoutcommit) (struct pnfs_layout_hdr *layoutid,
+                                      struct nfs4_layoutcommit_data *data);
+
 	void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
 				     struct xdr_stream *xdr,
 				     const struct nfs4_layoutcommit_args *args);

@@ -213,6 +216,7 @@ void pnfs_roc_release(struct inode *ino);
 void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
 bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
 void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
+void pnfs_cleanup_layoutcommit(struct inode *inode, struct nfs4_layoutcommit_data *data);
 int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
 int _pnfs_return_layout(struct inode *);
 int pnfs_ld_write_done(struct nfs_write_data *);

include/linux/nfs_xdr.h +1 -0

index a9c43ba..2c3ffda 100644
@@ -270,6 +270,7 @@ struct nfs4_layoutcommit_res {
 	struct nfs_fattr *fattr;
 	const struct nfs_server *server;
 	struct nfs4_sequence_res seq_res;
+	int status;
 };
 
 struct nfs4_layoutcommit_data {

Quoted 23 lines · Jim Rees

From: Peng Tao <[email protected]>

This gives layout driver a chance to cleanup structures they put in. Also ensure layoutcommit does not commit more than isize, as block layout driver may dirty pages beyond EOF.

Signed-off-by Andy Adamson <[email protected]>

[fixup layout header pointer for layoutcommit]

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/nfs4proc.c       |    1 +
 fs/nfs/nfs4xdr.c        |    3 ++-
 fs/nfs/pnfs.c           |   15 +++++++++++++++
 fs/nfs/pnfs.h           |    4 ++++
 include/linux/nfs_xdr.h |    1 +

 5 files changed, 23 insertions(+), 1 deletions(-)

fs/nfs/nfs4proc.c +0 -0

index 5246db8..e27a648 100644

fs/nfs/nfs4proc.c static void nfs4_layoutcommit_release(void *calldata) · line 5893

Quoted 4 lines

fs/nfs/nfs4proc.c +1 -0

@@ -5890,6 +5890,7 @@ static void nfs4_layoutcommit_release(void *calldata)
  {
       struct nfs4_layoutcommit_data *data = calldata;
 
+     pnfs_cleanup_layoutcommit(data->args.inode, data);

The layout driver better be passed the status on the done method rather than on release so that it can roll back on error.

Although it is quite complicated to roll back after permanent errors like NFS4ERR_BADLAYOUT where the client is really screwed and it essentially needs to redirty and rewrite the data (to the MDS to simplify the error handling path), rolling back from transient errors like NFS4ERR_DELAY should be fairly easy.

Benny

Quoted 94 lines

fs/nfs/nfs4proc.c +0 -0

       /* Matched by references in pnfs_set_layoutcommit */
       put_lseg(data->lseg);
       put_rpccred(data->cred);

fs/nfs/nfs4xdr.c +0 -0

index fdcbd8f..57295d1 100644
@@ -1963,7 +1963,7 @@ encode_layoutcommit(struct xdr_stream *xdr,
       *p++ = cpu_to_be32(OP_LAYOUTCOMMIT);
       /* Only whole file layouts */

      p = xdr_encode_hyper(p, 0); /* offset */
-     p = xdr_encode_hyper(p, NFS4_MAX_UINT64); /* length */
+     p = xdr_encode_hyper(p, args->lastbytewritten+1); /* length */ *p++ = cpu_to_be32(0); /* reclaim */
      p = xdr_encode_opaque_fixed(p, args->stateid.data, NFS4_STATEID_SIZE); *p++ = cpu_to_be32(1); /* newoffset = TRUE */

fs/nfs/nfs4xdr.c +1 -0

@@ -5467,6 +5467,7 @@ static int decode_layoutcommit(struct xdr_stream *xdr,
       int status;
 
       status = decode_op_hdr(xdr, OP_LAYOUTCOMMIT);
+     res->status = status;
       if (status)
               return status;
 

fs/nfs/pnfs.c +15 -0

index e693718..48a06a1 100644
@@ -1248,6 +1248,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
  {
       struct nfs_inode *nfsi = NFS_I(wdata->inode);
       loff_t end_pos = wdata->mds_offset + wdata->res.count;
+     loff_t isize = i_size_read(wdata->inode);
       bool mark_as_dirty = false;
 
       spin_lock(&nfsi->vfs_inode.i_lock);

@@ -1261,9 +1262,13 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
               dprintk("%s: Set layoutcommit for inode %lu ",
                       __func__, wdata->inode->i_ino);
       }
+     if (end_pos > isize)
+             end_pos = isize;
       if (end_pos > wdata->lseg->pls_end_pos)
               wdata->lseg->pls_end_pos = end_pos;
       spin_unlock(&nfsi->vfs_inode.i_lock);
+     dprintk("%s: lseg %p end_pos %llu\n",
+             __func__, wdata->lseg, wdata->lseg->pls_end_pos);
 
       /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
        * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */

@@ -1272,6 +1277,16 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
  }
  EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_cleanup_layoutcommit(struct inode *inode,
+                               struct nfs4_layoutcommit_data *data)
+{
+        struct nfs_server *nfss = NFS_SERVER(inode);
+
+        if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
+                nfss->pnfs_curr_ld->cleanup_layoutcommit(
+                                        NFS_I(inode)->layout, data);
+}
+
  void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
  {
       /* lseg refcounting handled directly in nfs_write_end */

fs/nfs/pnfs.h +4 -0

index 525ec55..5048898 100644
@@ -127,6 +127,9 @@ struct pnfs_layoutdriver_type {
                                    struct xdr_stream *xdr,
                                    const struct nfs4_layoutreturn_args *args);
 
+        void (*cleanup_layoutcommit) (struct pnfs_layout_hdr *layoutid,
+                                      struct nfs4_layoutcommit_data *data);
+
       void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
                                    struct xdr_stream *xdr,
                                    const struct nfs4_layoutcommit_args *args);

@@ -213,6 +216,7 @@ void pnfs_roc_release(struct inode *ino);
  void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
  bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
  void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
+void pnfs_cleanup_layoutcommit(struct inode *inode, struct nfs4_layoutcommit_data *data);
  int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
  int _pnfs_return_layout(struct inode *);
  int pnfs_ld_write_done(struct nfs_write_data *);

include/linux/nfs_xdr.h +0 -0

index a9c43ba..2c3ffda 100644

include/linux/nfs_xdr.h struct nfs4_layoutcommit_res { · line 276

Quoted 7 lines

include/linux/nfs_xdr.h +1 -0

@@ -270,6 +270,7 @@ struct nfs4_layoutcommit_res {
       struct nfs_fattr *fattr;
       const struct nfs_server *server;
       struct nfs4_sequence_res seq_res;
+     int status;
  };
 
  struct nfs4_layoutcommit_data {

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

From: Peng Tao <[email protected]>

This gives layout driver a chance to cleanup structures they put in. Also ensure layoutcommit does not commit more than isize, as block layout driver may dirty pages beyond EOF.

Signed-off-by Andy Adamson <[email protected]>

[fixup layout header pointer for layoutcommit]

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/nfs4proc.c       |    1 +
 fs/nfs/nfs4xdr.c        |    3 ++-
 fs/nfs/pnfs.c           |   15 +++++++++++++++
 fs/nfs/pnfs.h           |    4 ++++
 include/linux/nfs_xdr.h |    1 +
 5 files changed, 23 insertions(+), 1 deletions(-)

fs/nfs/nfs4proc.c +1 -0

index 5246db8..e27a648 100644
@@ -5890,6 +5890,7 @@ static void nfs4_layoutcommit_release(void *calldata)
 {
 	struct nfs4_layoutcommit_data *data = calldata;
 
+	pnfs_cleanup_layoutcommit(data->args.inode, data);
 	/* Matched by references in pnfs_set_layoutcommit */
 	put_lseg(data->lseg);
 	put_rpccred(data->cred);

fs/nfs/nfs4xdr.c +0 -0

index fdcbd8f..57295d1 100644

fs/nfs/nfs4xdr.c +0 -0

 	if (status)
 		return status;
 

fs/nfs/pnfs.c +15 -0

index e693718..48a06a1 100644
@@ -1248,6 +1248,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 {
 	struct nfs_inode *nfsi = NFS_I(wdata->inode);
 	loff_t end_pos = wdata->mds_offset + wdata->res.count;
+	loff_t isize = i_size_read(wdata->inode);
 	bool mark_as_dirty = false;
 
 	spin_lock(&nfsi->vfs_inode.i_lock);

@@ -1261,9 +1262,13 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 		dprintk("%s: Set layoutcommit for inode %lu ",
 			__func__, wdata->inode->i_ino);
 	}
+	if (end_pos > isize)
+		end_pos = isize;
 	if (end_pos > wdata->lseg->pls_end_pos)
 		wdata->lseg->pls_end_pos = end_pos;
 	spin_unlock(&nfsi->vfs_inode.i_lock);
+	dprintk("%s: lseg %p end_pos %llu\n",
+		__func__, wdata->lseg, wdata->lseg->pls_end_pos);
 
 	/* if pnfs_layoutcommit_inode() runs between inode locks, the next one
 	 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */

@@ -1272,6 +1277,16 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
 }
 EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_cleanup_layoutcommit(struct inode *inode,
+                               struct nfs4_layoutcommit_data *data)
+{
+        struct nfs_server *nfss = NFS_SERVER(inode);
+
+        if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
+                nfss->pnfs_curr_ld->cleanup_layoutcommit(
+                                        NFS_I(inode)->layout, data);
+}
+
 void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
 {
 	/* lseg refcounting handled directly in nfs_write_end */

fs/nfs/pnfs.h +0 -0

index 525ec55..5048898 100644

fs/nfs/pnfs.h +1 -0

 	void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid,
 				     struct xdr_stream *xdr,
 				     const struct nfs4_layoutcommit_args *args);

@@ -213,6 +216,7 @@ void pnfs_roc_release(struct inode *ino);
 void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
 bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
 void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
+void pnfs_cleanup_layoutcommit(struct inode *inode, struct nfs4_layoutcommit_data *data);
 int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
 int _pnfs_return_layout(struct inode *);
 int pnfs_ld_write_done(struct nfs_write_data *);

include/linux/nfs_xdr.h +0 -0

index a9c43ba..2c3ffda 100644

include/linux/nfs_xdr.h +0 -0

 };
 
 struct nfs4_layoutcommit_data {

Quoted 5 lines · Jim Rees

From: Peng Tao <[email protected]>

This gives layout driver a chance to cleanup structures they put in. Also ensure layoutcommit does not commit more than isize, as block layout driver may dirty pages beyond EOF.

let's separate the latter matter into a different patch so we can discuss the problem and the solution orthogonally to cleanup_layoutcommit.

Quoted 32 lines

Signed-off-by Andy Adamson <[email protected]>

[fixup layout header pointer for layoutcommit]

Signed-off-by Benny Halevy <[email protected]>

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/nfs4proc.c       |    1 +
 fs/nfs/nfs4xdr.c        |    3 ++-
 fs/nfs/pnfs.c           |   15 +++++++++++++++
 fs/nfs/pnfs.h           |    4 ++++
 include/linux/nfs_xdr.h |    1 +

 5 files changed, 23 insertions(+), 1 deletions(-)

fs/nfs/nfs4proc.c +1 -0

index 5246db8..e27a648 100644
@@ -5890,6 +5890,7 @@ static void nfs4_layoutcommit_release(void *calldata)
  {
       struct nfs4_layoutcommit_data *data = calldata;
 
+     pnfs_cleanup_layoutcommit(data->args.inode, data);
       /* Matched by references in pnfs_set_layoutcommit */
       put_lseg(data->lseg);
       put_rpccred(data->cred);

fs/nfs/nfs4xdr.c +0 -0

index fdcbd8f..57295d1 100644
@@ -1963,7 +1963,7 @@ encode_layoutcommit(struct xdr_stream *xdr,
       *p++ = cpu_to_be32(OP_LAYOUTCOMMIT);
       /* Only whole file layouts */

Quoted 3 lines

      p = xdr_encode_hyper(p, 0); /* offset */
-     p = xdr_encode_hyper(p, NFS4_MAX_UINT64); /* length */
+     p = xdr_encode_hyper(p, args->lastbytewritten+1); /* length */

This is unrelated to this particular patch and it should be discussed separately. (and dropped altogether :)

Quoted 6 lines

      *p++ = cpu_to_be32(0); /* reclaim */
      p = xdr_encode_opaque_fixed(p, args->stateid.data, NFS4_STATEID_SIZE); *p++ = cpu_to_be32(1); /* newoffset = TRUE */

fs/nfs/nfs4xdr.c +0 -0

@@ -5467,6 +5467,7 @@ static int decode_layoutcommit(struct xdr_stream *xdr,
       int status;
 

fs/nfs/nfs4xdr.c static int decode_layoutcommit(struct xdr_stream *xdr, · line 5470

Quoted 2 lines

fs/nfs/nfs4xdr.c +1 -0

       status = decode_op_hdr(xdr, OP_LAYOUTCOMMIT);
+     res->status = status;

What is res->status used for?

Quoted 55 lines

fs/nfs/nfs4xdr.c +0 -0

       if (status)
               return status;
 

fs/nfs/pnfs.c +15 -0

index e693718..48a06a1 100644
@@ -1248,6 +1248,7 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
  {
       struct nfs_inode *nfsi = NFS_I(wdata->inode);
       loff_t end_pos = wdata->mds_offset + wdata->res.count;
+     loff_t isize = i_size_read(wdata->inode);
       bool mark_as_dirty = false;
 
       spin_lock(&nfsi->vfs_inode.i_lock);

@@ -1261,9 +1262,13 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
               dprintk("%s: Set layoutcommit for inode %lu ",
                       __func__, wdata->inode->i_ino);
       }
+     if (end_pos > isize)
+             end_pos = isize;
       if (end_pos > wdata->lseg->pls_end_pos)
               wdata->lseg->pls_end_pos = end_pos;
       spin_unlock(&nfsi->vfs_inode.i_lock);
+     dprintk("%s: lseg %p end_pos %llu\n",
+             __func__, wdata->lseg, wdata->lseg->pls_end_pos);
 
       /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
        * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */

@@ -1272,6 +1277,16 @@ pnfs_set_layoutcommit(struct nfs_write_data *wdata)
  }
  EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
 
+void pnfs_cleanup_layoutcommit(struct inode *inode,
+                               struct nfs4_layoutcommit_data *data)
+{
+        struct nfs_server *nfss = NFS_SERVER(inode);
+
+        if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
+                nfss->pnfs_curr_ld->cleanup_layoutcommit(
+                                        NFS_I(inode)->layout, data);
+}
+
  void pnfs_free_fsdata(struct pnfs_fsdata *fsdata)
  {
       /* lseg refcounting handled directly in nfs_write_end */

fs/nfs/pnfs.h +0 -0

index 525ec55..5048898 100644
@@ -127,6 +127,9 @@ struct pnfs_layoutdriver_type {

                                   struct xdr_stream *xdr,it:
                                   const struct nfs4_layoutreturn_args *args);

Quoted 3 lines

+        void (*cleanup_layoutcommit) (struct pnfs_layout_hdr *layoutid,
+                                      struct nfs4_layoutcommit_data *data); +

nit: whitespace cleanup required...

Quoted 16 lines

      void (*encode_layoutcommit) (struct pnfs_layout_hdr *layoutid, struct xdr_stream *xdr,
                                   const struct nfs4_layoutcommit_args *args);

fs/nfs/pnfs.h +1 -0

@@ -213,6 +216,7 @@ void pnfs_roc_release(struct inode *ino);
  void pnfs_roc_set_barrier(struct inode *ino, u32 barrier);
  bool pnfs_roc_drain(struct inode *ino, u32 *barrier);
  void pnfs_set_layoutcommit(struct nfs_write_data *wdata);
+void pnfs_cleanup_layoutcommit(struct inode *inode, struct nfs4_layoutcommit_data *data);
  int pnfs_layoutcommit_inode(struct inode *inode, bool sync);
  int _pnfs_return_layout(struct inode *);
  int pnfs_ld_write_done(struct nfs_write_data *);

include/linux/nfs_xdr.h +0 -0

index a9c43ba..2c3ffda 100644

include/linux/nfs_xdr.h struct nfs4_layoutcommit_res { · line 273

Quoted 4 lines

include/linux/nfs_xdr.h +1 -0

@@ -270,6 +270,7 @@ struct nfs4_layoutcommit_res {
       struct nfs_fattr *fattr;
       const struct nfs_server *server;
       struct nfs4_sequence_res seq_res;
+     int status;

This seems to be unused in this patch...

Benny

include/linux/nfs_xdr.h struct nfs4_layoutcommit_res { · line 276

Quoted 3 lines

include/linux/nfs_xdr.h +0 -0

  };
 
  struct nfs4_layoutcommit_data {

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html