pnfs post wave2 changes
Subsystems (2)
cover pnfs post wave2 changes
Here are the list of patches I'd like to reintroduce into the post-submit pnfs branch, after Fred's wave2 patches to bring layoutreturn and layoutcommit and "forgetless" cb_layoutrecall functionality back into the pnfs branch:
[PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions" [PATCH 2/9] Revert "pnfs-submit: Turn off layoutcommits"
[PATCH 3/9] Revert "pnfs-submit: wave2: remove all LAYOUTRETURN code" [PATCH 4/9] Revert "pnfs-submit: wave2: Remove LAYOUTRETURN from return on close" [PATCH 5/9] FIXME: roc should return layout on last close
[PATCH 6/9] Revert "pnfs-submit: wave2: remove cl_layoutrecalls list" [PATCH 7/9] Revert "pnfs-submit: wave2: Pull out all recall initiated LAYOUTRETURNS" [PATCH 8/9] Revert "pnfs-submit: wave2: Don't wait in layoutget"
[PATCH 9/9] Revert "pnfs-submit: wave2: check that partial LAYOUTGET return is ignored"
Benny
PATCH 1/9 Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
This reverts commit 19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.
--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)
include/linux/nfs4.h +1 -0
@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,
include/linux/nfs_xdr.h +23 -0
@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status; +}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;
Signature · 1 line
1.7.2.3
PATCH 2/9 Revert "pnfs-submit: Turn off layoutcommits"
In reply to Benny HalevyThis reverts commit 1120c1ee8209679aa187248917875665dd8521a0.
--- fs/nfs/nfs4proc.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
fs/nfs/nfs4proc.c +1 -0
@@ -3098,6 +3098,7 @@ static void pnfs4_update_write_done(struct nfs_inode *nfsi, struct nfs_write_dat {#ifdef CONFIG_NFS_V4_1 pnfs_update_last_write (nfsi, data->args.offset, data->res.count); +pnfs_need_layoutcommit (nfsi, data->args.context);#endif /* CONFIG_NFS_V4_1 */ }
Signature · 1 line
1.7.2.3
PATCH 3/9 Revert "pnfs-submit: wave2: remove all LAYOUTRETURN code"
In reply to Benny HalevyThis reverts commit 7fc5d08da0e6bcff7363199c3238f8f807663ab2.
--- fs/nfs/nfs4proc.c | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfs/nfs4xdr.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++++ fs/nfs/pnfs.c | 64 +++++++++++++++++++++++++++- fs/nfs/pnfs.h | 2 + 4 files changed, 292 insertions(+), 3 deletions(-)
fs/nfs/nfs4proc.c +118 -0
@@ -5554,6 +5554,124 @@ out:return 0 ; } +static void +nfs4_layoutreturn_prepare (struct rpc_task *task,void *calldata) +{ +struct nfs4_layoutreturn *lrp = calldata; + +dprintk ("--> %s\n" , __func__); +if (lrp->args.return_type ==RETURN_FILE ) { +struct nfs_inode *nfsi =NFS_I (lrp->args.inode); + +if (pnfs_return_layout_barrier (nfsi, &lrp->args.range)) { +dprintk ("%s: waiting on barrier\n" , __func__); +rpc_sleep_on (&nfsi->lo_rpcwaitq, task,NULL ); +return ; + } + } +if (nfs41_setup_sequence (lrp->clp->cl_session, &lrp->args.seq_args, + &lrp->res.seq_res,0 , task)) +return ; +rpc_call_start (task); +} + +static void nfs4_layoutreturn_done (struct rpc_task *task,void *calldata) +{ +struct nfs4_layoutreturn *lrp = calldata; +struct nfs_server *server; + +dprintk ("--> %s\n" , __func__); + +if (!nfs4_sequence_done (task, &lrp->res.seq_res)) +return ; + +if (lrp->args.return_type ==RETURN_FILE ) + server =NFS_SERVER (lrp->args.inode); +else + server =NULL ; +if (nfs4_async_handle_error (task, server,NULL , lrp->clp) == -EAGAIN ) { +nfs_restart_rpc (task, lrp->clp); +return ; + } +if ((task->tk_status ==0 ) && (lrp->args.return_type ==RETURN_FILE )) { +struct pnfs_layout_hdr *lo =NFS_I (lrp->args.inode)->layout; + +spin_lock (&lo->inode->i_lock); +if (lrp->res.lrs_present) +pnfs_set_layout_stateid (lo, &lrp->res.stateid, true); +else +BUG_ON (!list_empty (&lo->segs)); +spin_unlock (&lo->inode->i_lock); + } +dprintk ("<-- %s\n" , __func__); +} + +static void nfs4_layoutreturn_release (void *calldata) +{ +struct nfs4_layoutreturn *lrp = calldata; + +dprintk ("--> %s return_type %d\n" , __func__, lrp->args.return_type); +if (lrp->args.return_type ==RETURN_FILE ) { +struct inode *ino = lrp->args.inode; +struct pnfs_layout_hdr *lo =NFS_I (ino)->layout; + +spin_lock (&ino->i_lock); + lo->plh_block_lgets--; +atomic_dec (&lo->plh_outstanding); +spin_unlock (&ino->i_lock); +put_layout_hdr (ino); + } +kfree (calldata); +dprintk ("<-- %s\n" , __func__); +} + +static const struct rpc_call_ops nfs4_layoutreturn_call_ops = { + .rpc_call_prepare = nfs4_layoutreturn_prepare, + .rpc_call_done = nfs4_layoutreturn_done, + .rpc_release = nfs4_layoutreturn_release, +}; + +int nfs4_proc_layoutreturn (struct nfs4_layoutreturn *lrp,bool issync) +{ +struct rpc_task *task; +struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTRETURN ], + .rpc_argp = &lrp->args, + .rpc_resp = &lrp->res, + }; +struct rpc_task_setup task_setup_data = { + .rpc_client = lrp->clp->cl_rpcclient, + .rpc_message = &msg, + .callback_ops = &nfs4_layoutreturn_call_ops, + .callback_data = lrp, + .flags =RPC_TASK_ASYNC , + }; +int status =0 ; + +dprintk ("--> %s\n" , __func__); +if (lrp->args.return_type ==RETURN_FILE ) { +struct pnfs_layout_hdr *lo =NFS_I (lrp->args.inode)->layout; +/* FIXME we should test for BULK here */ +spin_lock (&lo->inode->i_lock); +BUG_ON (lo->plh_block_lgets ==0 ); +atomic_inc (&lo->plh_outstanding); +spin_unlock (&lo->inode->i_lock); + } + task =rpc_run_task (&task_setup_data); +if (IS_ERR (task)) +return PTR_ERR (task); +if (!issync) + goto out; + status =nfs4_wait_for_completion_rpc_task (task); +if (status !=0 ) + goto out; + status = task->tk_status; +out: +dprintk ("<-- %s\n" , __func__); +rpc_put_task (task); +return status; +} +static int _nfs4_proc_getdeviceinfo (struct nfs_server *server,struct pnfs_device *pdev) {
fs/nfs/nfs4xdr.c +111 -0
153 lines not on this page · raw message
fs/nfs/pnfs.c +61 -3
80 lines not on this page · raw message
fs/nfs/pnfs.h +2 -0
@@ -182,6 +182,7 @@ extern int nfs4_proc_getdeviceinfo(struct nfs_server *server,extern int nfs4_proc_layoutget (struct nfs4_layoutget *lgp);extern int nfs4_proc_layoutcommit (struct nfs4_layoutcommit_data *data,int issync); +extern int nfs4_proc_layoutreturn (struct nfs4_layoutreturn *lrp,bool wait);/* pnfs.c */ void get_layout_hdr (struct pnfs_layout_hdr *lo);
@@ -191,6 +192,7 @@ bool should_free_lseg(struct pnfs_layout_range *lseg_range,struct pnfs_layout_segment *pnfs_update_layout (struct inode *ino,struct nfs_open_context *ctx,enum pnfs_iomode access_type); +bool pnfs_return_layout_barrier (struct nfs_inode *,struct pnfs_layout_range *);int _pnfs_return_layout (struct inode *,struct pnfs_layout_range *,bool wait);void set_pnfs_layoutdriver (struct nfs_server *,u32 id);void unset_pnfs_layoutdriver (struct nfs_server *);
Signature · 1 line
1.7.2.3
PATCH 4/9 Revert "pnfs-submit: wave2: Remove LAYOUTRETURN from return on close"
In reply to Benny HalevyThis reverts commit 5611feb54b4c0225f0924c6b059f281d45915b24.
--- fs/nfs/nfs4_fs.h | 2 +- fs/nfs/nfs4proc.c | 16 ++------- fs/nfs/nfs4state.c | 16 +++++++- fs/nfs/pnfs.c | 103 +++++----------------------------------------------- fs/nfs/pnfs.h | 38 ++++++------------- 5 files changed, 39 insertions(+), 136 deletions(-)
fs/nfs/nfs4_fs.h +1 -1
@@ -242,7 +242,7 @@ extern int nfs4_proc_async_renew(struct nfs_client *, struct rpc_cred *);extern int nfs4_proc_renew (struct nfs_client *,struct rpc_cred *);extern int nfs4_init_clientid (struct nfs_client *,struct rpc_cred *);extern int nfs41_init_clientid (struct nfs_client *,struct rpc_cred *); -extern int nfs4_do_close (struct path *path,struct nfs4_state *state,gfp_t gfp_mask,int wait,bool roc); +extern int nfs4_do_close (struct path *path,struct nfs4_state *state,gfp_t gfp_mask,int wait);extern int nfs4_server_capabilities (struct nfs_server *server,struct nfs_fh *fhandle);extern int nfs4_proc_fs_locations (struct inode *dir,const struct qstr *name,struct nfs4_fs_locations *fs_locations,struct page *page);
fs/nfs/nfs4proc.c +3 -13
@@ -1841,8 +1841,6 @@ struct nfs4_closedata {struct nfs_closeres res;struct nfs_fattr fattr;unsigned long timestamp; -bool roc; -u32 roc_barrier; };static void nfs4_free_closedata (void *data)
@@ -1850,7 +1848,6 @@ static void nfs4_free_closedata(void *data)struct nfs4_closedata *calldata = data;struct nfs4_state_owner *sp = calldata->state->owner; -pnfs_roc_release (calldata->roc, calldata->state->inode);nfs4_put_open_state (calldata->state);nfs_free_seqid (calldata->arg.seqid);nfs4_put_state_owner (sp);
@@ -1883,8 +1880,6 @@ static void nfs4_close_done(struct rpc_task *task, void *data)*/ switch (task->tk_status) {case 0 : -pnfs_roc_set_barrier (calldata->roc, state->inode, - calldata->roc_barrier);nfs_set_open_stateid (state, &calldata->res.stateid,0 );renew_lease (server, calldata->timestamp); nfs4_close_clear_stateid_flags(state,
@@ -1937,11 +1932,8 @@ static void nfs4_close_prepare(struct rpc_task *task, void *data)return ; } -if (calldata->arg.fmode ==0 ) { +if (calldata->arg.fmode ==0 ) task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE ]; -pnfs_roc_drain (calldata->roc, state->inode, - &calldata->roc_barrier, task); - }nfs_fattr_init (calldata->res.fattr); calldata->timestamp = jiffies;
@@ -1969,7 +1961,7 @@ static const struct rpc_call_ops nfs4_close_ops = {* * NOTE: Caller must be holding the sp->so_owner semaphore! */ -int nfs4_do_close (struct path *path,struct nfs4_state *state,gfp_t gfp_mask,int wait,bool roc) +int nfs4_do_close (struct path *path,struct nfs4_state *state,gfp_t gfp_mask,int wait) {struct nfs_server *server =NFS_SERVER (state->inode);struct nfs4_closedata *calldata;
@@ -2004,7 +1996,6 @@ int nfs4_do_close(struct path *path, struct nfs4_state *state, gfp_t gfp_mask, i calldata->res.fattr = &calldata->fattr; calldata->res.seqid = calldata->arg.seqid; calldata->res.server = server; - calldata->roc = roc;path_get (path); calldata->path = *path;
@@ -2022,7 +2013,6 @@ int nfs4_do_close(struct path *path, struct nfs4_state *state, gfp_t gfp_mask, i out_free_calldata:kfree (calldata); out: -pnfs_roc_release (roc, state->inode);nfs4_put_open_state (state);nfs4_put_state_owner (sp);return status;
@@ -5619,7 +5609,7 @@ static void nfs4_layoutreturn_release(void *calldata) lo->plh_block_lgets--;atomic_dec (&lo->plh_outstanding);spin_unlock (&ino->i_lock); -put_layout_hdr (ino); +put_layout_hdr (lo); }kfree (calldata);dprintk ("<-- %s\n" , __func__);
fs/nfs/nfs4state.c +14 -2
@@ -619,9 +619,21 @@ static void __nfs4_close(struct path *path, struct nfs4_state *state,nfs4_put_open_state (state);nfs4_put_state_owner (owner); }else { -bool roc =pnfs_roc (state->inode); +u32 roc_iomode; +struct nfs_inode *nfsi =NFS_I (state->inode); + +if (has_layout (nfsi) && + (roc_iomode =pnfs_layout_roc_iomode (nfsi)) !=0 ) { +struct pnfs_layout_range range = { + .iomode = roc_iomode, + .offset =0 , + .length =NFS4_MAX_UINT64 , + }; + +pnfs_return_layout (state->inode, &range, wait); + } -nfs4_do_close (path, state, gfp_mask, wait, roc); +nfs4_do_close (path, state, gfp_mask, wait); } }
fs/nfs/pnfs.c +9 -94
127 lines not on this page · raw message
fs/nfs/pnfs.h +12 -26
@@ -34,7 +34,6 @@enum {NFS_LSEG_VALID =0 ,/* cleared when lseg is recalled/returned */ -NFS_LSEG_ROC ,/* roc bit received from server */ };struct pnfs_layout_segment {
@@ -60,7 +59,6 @@ enum {NFS_LAYOUT_RW_FAILED ,/* get rw layout failed stop trying */ NFS_LAYOUT_BULK_RECALL ,/* bulk recall affecting layout */ NFS_LAYOUT_NEED_LCOMMIT, /* LAYOUTCOMMIT needed */ - NFS_LAYOUT_ROC, /* some lseg had roc bit set */ };/* Per-layout driver specific registration structure */
@@ -103,6 +101,7 @@ struct pnfs_layout_hdr {struct list_head layouts;/* other client layouts */ struct list_head plh_bulk_recall;/* clnt list of bulk recalls */ struct list_head segs;/* layout segments list */ +int roc_iomode;/* return on close iomode, 0=none */ nfs4_stateid stateid;atomic_t plh_outstanding;/* number of RPCs out */ unsigned long plh_block_lgets;/* block LAYOUTGET if >0 */
@@ -223,11 +222,6 @@ int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,void nfs4_asynch_forget_layouts (struct pnfs_layout_hdr *lo,struct pnfs_layout_range *range,struct list_head *tmp_list); -bool pnfs_roc (struct inode *ino); -void pnfs_roc_release (bool needed,struct inode *ino); -void pnfs_roc_set_barrier (bool needed,struct inode *ino,u32 barrier); -void pnfs_roc_drain (bool needed,struct inode *ino,u32 *barrier, -struct rpc_task *task);static inline bool has_layout (struct nfs_inode *nfsi)
@@ -253,6 +247,14 @@ static inline int pnfs_enabled_sb(struct nfs_server *nfss)return nfss->pnfs_curr_ld !=NULL ; } +/* Should the pNFS client commit and return the layout on close +*/ +static inline int +pnfs_layout_roc_iomode (struct nfs_inode *nfsi) +{ +return nfsi->layout->roc_iomode; +} +static inline int pnfs_return_layout (struct inode *ino,struct pnfs_layout_range *range,bool wait)
@@ -336,26 +338,10 @@ static inline int pnfs_layoutcommit_inode(struct inode *inode, int sync)return 0 ; } -static inline bool -pnfs_roc (struct inode *ino) -{ -return false; -} - -static inline void -pnfs_roc_release (bool needed,struct inode *ino) -{ -} - -static inline void -pnfs_roc_set_barrier (bool needed,struct inode *ino,u32 barrier) -{ -} - -static inline void -pnfs_roc_drain (bool needed,struct inode *ino,u32 *barrier, -struct rpc_task *task) +static inline int +pnfs_layout_roc_iomode (struct nfs_inode *nfsi) { +return 0 ; } static inline int pnfs_return_layout(struct inode *ino,
Signature · 1 line
1.7.2.3
PATCH 5/9 FIXME: roc should return layout on last close
In reply to Benny Halevyrather than on any close.
--- fs/nfs/nfs4state.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
fs/nfs/nfs4state.c +1 -0
@@ -622,6 +622,7 @@ static void __nfs4_close(struct path *path, struct nfs4_state *state,u32 roc_iomode;struct nfs_inode *nfsi =NFS_I (state->inode); +/* FIXME: should return the layout only on last close */ if (has_layout (nfsi) && (roc_iomode =pnfs_layout_roc_iomode (nfsi)) !=0 ) {struct pnfs_layout_range range = {
Signature · 1 line
1.7.2.3
PATCH 6/9 Revert "pnfs-submit: wave2: remove cl_layoutrecalls list"
In reply to Benny HalevyThis reverts commit a7324f1761109d4563dc71b9e03e71440f9f1c4a.
Conflicts:
fs/nfs/callback_proc.c
fs/nfs/pnfs.c
--- fs/nfs/callback.h | 4 +- fs/nfs/callback_proc.c | 165 +++++++++++++++++++++++++++++++++++++++------ fs/nfs/client.c | 1 + fs/nfs/nfs4_fs.h | 1 - fs/nfs/nfs4proc.c | 10 ++- fs/nfs/pnfs.c | 37 ++++++---- fs/nfs/pnfs.h | 13 +++- include/linux/nfs_fs_sb.h | 4 + 8 files changed, 193 insertions(+), 42 deletions(-)
fs/nfs/callback.h +3 -1
@@ -154,7 +154,6 @@ struct cb_layoutrecallargs {union {struct {struct nfs_fh cbl_fh; -struct inode *cbl_inode;struct pnfs_layout_range cbl_range;nfs4_stateid cbl_stateid; };
@@ -165,6 +164,9 @@ struct cb_layoutrecallargs {extern unsigned nfs4_callback_layoutrecall (struct cb_layoutrecallargs *args,void *dummy,struct cb_process_state *cps); +extern bool matches_outstanding_recall (struct inode *ino, +struct pnfs_layout_range *range); +extern void notify_drained (struct nfs_client *clp,u64 mask);static inline void put_session_client (struct nfs4_session *session) {
fs/nfs/callback_proc.c +143 -22
225 lines not on this page · raw message
fs/nfs/client.c +1 -0
@@ -158,6 +158,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_ clp->cl_machine_cred = cred;#if defined(CONFIG_NFS_V4_1 )INIT_LIST_HEAD (&clp->cl_layouts); +INIT_LIST_HEAD (&clp->cl_layoutrecalls);#endif nfs_fscache_get_client_cookie (clp);
fs/nfs/nfs4_fs.h +0 -1
@@ -44,7 +44,6 @@ enum nfs4_client_state {NFS4CLNT_RECLAIM_REBOOT ,NFS4CLNT_RECLAIM_NOGRACE ,NFS4CLNT_DELEGRETURN , -NFS4CLNT_LAYOUTRECALL ,NFS4CLNT_SESSION_RESET , NFS4CLNT_RECALL_SLOT, };
fs/nfs/nfs4proc.c +8 -2
@@ -5378,9 +5378,14 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)dprintk ("--> %s\n" , __func__); -if (!nfs4_sequence_done (task, &lgp->res.seq_res)) +if (!nfs4_sequence_done (task, &lgp->res.seq_res)) { +/* layout code relies on fact that in this case +* code falls back to tk_action=call_start, but not +* back to rpc_prepare_task, to keep plh_outstanding +* correct. +*/ return ; - + }switch (task->tk_status) {case 0 :break ;
@@ -5402,6 +5407,7 @@ static void nfs4_layoutget_release(void *calldata)struct nfs4_layoutget *lgp = calldata;dprintk ("--> %s\n" , __func__); +put_layout_hdr (NFS_I (lgp->args.inode)->layout);if (lgp->res.layout.buf !=NULL )free_page ((unsigned long ) lgp->res.layout.buf);put_nfs_open_context (lgp->args.ctx);
fs/nfs/pnfs.c +22 -15
@@ -277,17 +277,17 @@ init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)smp_mb ();set_bit (NFS_LSEG_VALID , &lseg->pls_flags); lseg->layout = lo; - lseg->pls_recall_count =0 ; + lseg->pls_notify_mask =0 ; }static void free_lseg (struct pnfs_layout_segment *lseg) {struct inode *ino = lseg->layout->inode; -int count = lseg->pls_recall_count; +u64 mask = lseg->pls_notify_mask;BUG_ON (atomic_read (&lseg->pls_refcount) !=0 );NFS_SERVER (ino)->pnfs_curr_ld->free_lseg (lseg); -atomic_sub (count, &NFS_SERVER (ino)->nfs_client->cl_recall_count); +notify_drained (NFS_SERVER (ino)->nfs_client, mask);/* Matched by get_layout_hdr_locked in pnfs_insert_layout */ put_layout_hdr (NFS_I (ino)->layout); }
@@ -544,8 +544,10 @@ send_layoutget(struct pnfs_layout_hdr *lo,BUG_ON (ctx ==NULL ); lgp =kzalloc (sizeof (*lgp),GFP_KERNEL ); -if (lgp ==NULL ) +if (lgp ==NULL ) { +put_layout_hdr (lo);return NULL ; + } lgp->args.minlength =NFS4_MAX_UINT64 ; lgp->args.maxcount =PNFS_LAYOUT_MAXSIZE ; lgp->args.range.iomode = range->iomode;
@@ -569,6 +571,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,void nfs4_asynch_forget_layouts (struct pnfs_layout_hdr *lo,struct pnfs_layout_range *range, +int notify_bit,atomic_t *notify_count,struct list_head *tmp_list) {struct pnfs_layout_segment *lseg, *tmp;
@@ -576,8 +579,8 @@ void nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,assert_spin_locked (&lo->inode->i_lock);list_for_each_entry_safe (lseg, tmp, &lo->segs, fi_list)if (should_free_lseg (&lseg->range, range)) { - lseg->pls_recall_count++; -atomic_inc (&NFS_SERVER (lo->inode)->nfs_client->cl_recall_count); + lseg->pls_notify_mask |= (1 << notify_bit); +atomic_inc (notify_count);mark_lseg_invalid (lseg, tmp_list); } }
@@ -833,6 +836,13 @@ pnfs_update_layout(struct inode *ino,if (!pnfs_enabled_sb (NFS_SERVER (ino)))return NULL ; +spin_lock (&clp->cl_lock); +if (matches_outstanding_recall (ino, &arg)) { +dprintk ("%s matches recall, use MDS\n" , __func__); +spin_unlock (&clp->cl_lock); +return NULL ; + } +spin_unlock (&clp->cl_lock);spin_lock (&ino->i_lock); lo =pnfs_find_alloc_layout (ino);if (lo ==NULL ) {
@@ -840,12 +850,6 @@ pnfs_update_layout(struct inode *ino, goto out_unlock; } -/* Do we even need to bother with this? */ -if (test_bit (NFS4CLNT_LAYOUTRECALL , &clp->cl_state) || -test_bit (NFS_LAYOUT_BULK_RECALL , &lo->plh_flags)) { -dprintk ("%s matches recall, use MDS\n" , __func__); - goto out_unlock; - }/* Check to see if the layout for the given range already exists */ lseg =pnfs_find_lseg (lo, &arg);if (lseg)
@@ -883,7 +887,7 @@ pnfs_update_layout(struct inode *ino,spin_unlock (&ino->i_lock); }atomic_dec (&lo->plh_outstanding); -put_layout_hdr (lo); +spin_unlock (&ino->i_lock); out:dprintk ("%s end, state 0x%lx lseg %p\n" , __func__, nfsi->layout->plh_flags, lseg);
@@ -927,11 +931,14 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) }spin_lock (&ino->i_lock); -if (test_bit (NFS4CLNT_LAYOUTRECALL , &clp->cl_state) || -test_bit (NFS_LAYOUT_BULK_RECALL , &lo->plh_flags)) { +/* decrement needs to be done before call to pnfs_layoutget_blocked */ +spin_lock (&clp->cl_lock); +if (matches_outstanding_recall (ino, &res->range)) { +spin_unlock (&clp->cl_lock);dprintk ("%s forget reply due to recall\n" , __func__); goto out_forget_reply; } +spin_unlock (&clp->cl_lock);if (pnfs_layoutgets_blocked (lo, &res->stateid,1 )) {dprintk ("%s forget reply due to state\n" , __func__);
fs/nfs/pnfs.h +12 -1
@@ -31,6 +31,7 @@#define FS_NFS_PNFS_H #include <linux/nfs_page.h> +#include "callback.h" enum {NFS_LSEG_VALID =0 ,/* cleared when lseg is recalled/returned */
@@ -42,7 +43,7 @@ struct pnfs_layout_segment {atomic_t pls_refcount;unsigned long pls_flags;struct pnfs_layout_hdr *layout; -int pls_recall_count; +u64 pls_notify_mask; };enum pnfs_try_status {
@@ -126,6 +127,15 @@ struct pnfs_device {unsigned int pglen; }; +struct pnfs_cb_lrecall_info { +struct list_head pcl_list;/* hook into cl_layoutrecalls list */ +atomic_t pcl_count; +int pcl_notify_bit; +struct nfs_client *pcl_clp; +struct inode *pcl_ino; +struct cb_layoutrecallargs pcl_args; +}; +/* * Device ID RCU cache. A device ID is unique per client ID and layout type. */
@@ -221,6 +231,7 @@ int pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct nfs4_state *open_state);void nfs4_asynch_forget_layouts (struct pnfs_layout_hdr *lo,struct pnfs_layout_range *range, +int notify_bit,atomic_t *notify_count,struct list_head *tmp_list); static inline bool
include/linux/nfs_fs_sb.h +4 -0
@@ -84,6 +84,10 @@ struct nfs_client {struct nfs4_session *cl_session;/* sharred session */ struct list_head cl_layouts;atomic_t cl_recall_count;/* no. of lsegs in recall */ +struct list_head cl_layoutrecalls; +unsigned long cl_cb_lrecall_count; +#define PNFS_MAX_CB_LRECALLS (64) +atomic_t *cl_drain_notification[PNFS_MAX_CB_LRECALLS ];struct pnfs_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */
Signature · 1 line
1.7.2.3
PATCH 7/9 Revert "pnfs-submit: wave2: Pull out all recall initiated LAYOUTRETURNS"
In reply to Benny HalevyThis reverts commit fea1ffe711454410ba5277115c9aeaf814681f4a.
Conflicts:
fs/nfs/callback_proc.c
--- fs/nfs/callback.h | 5 ++ fs/nfs/callback_proc.c | 103 ++++++++++++++++++++++++++++++++++------------- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4state.c | 4 ++ 4 files changed, 84 insertions(+), 29 deletions(-)
fs/nfs/callback.h +5 -0
@@ -167,6 +167,7 @@ extern unsigned nfs4_callback_layoutrecall(extern bool matches_outstanding_recall (struct inode *ino,struct pnfs_layout_range *range);extern void notify_drained (struct nfs_client *clp,u64 mask); +extern void nfs_client_return_layouts (struct nfs_client *clp);static inline void put_session_client (struct nfs4_session *session) {
@@ -182,6 +183,10 @@ find_client_from_cps(struct cb_process_state *cps, struct sockaddr *addr)#else /* CONFIG_NFS_V4_1 */ +static inline void nfs_client_return_layouts (struct nfs_client *clp) +{ +} +static inline struct nfs_client *find_client_from_cps (struct cb_process_state *cps,struct sockaddr *addr) {
fs/nfs/callback_proc.c +74 -29
131 lines not on this page · raw message
fs/nfs/nfs4_fs.h +1 -0
@@ -46,6 +46,7 @@ enum nfs4_client_state {NFS4CLNT_DELEGRETURN ,NFS4CLNT_SESSION_RESET ,NFS4CLNT_RECALL_SLOT , + NFS4CLNT_LAYOUT_RECALL, }; enum nfs4_session_state {
fs/nfs/nfs4state.c +4 -0
@@ -1577,6 +1577,10 @@ static void nfs4_state_manager(struct nfs_client *clp)nfs_client_return_marked_delegations (clp);continue ; } +if (test_and_clear_bit (NFS4CLNT_LAYOUT_RECALL , &clp->cl_state)) { +nfs_client_return_layouts (clp); +continue ; + }/* Recall session slots */ if (test_and_clear_bit (NFS4CLNT_RECALL_SLOT , &clp->cl_state) &&nfs4_has_session (clp)) {
Signature · 1 line
1.7.2.3
PATCH 8/9 Revert "pnfs-submit: wave2: Don't wait in layoutget"
In reply to Benny HalevyThis reverts commit 9628f174680a6ec76aa6e1afa8678631a5efd39b.
Conflicts:
fs/nfs/nfs4proc.c
fs/nfs/pnfs.c
--- fs/nfs/callback_proc.c | 4 +++ fs/nfs/client.c | 2 + fs/nfs/inode.c | 1 + fs/nfs/nfs4proc.c | 47 ++++++++++++++++++++++++++++++++++- fs/nfs/pnfs.c | 58 +++++++++++++++++++++------------------------ fs/nfs/pnfs.h | 1 + include/linux/nfs_fs.h | 1 + include/linux/nfs_fs_sb.h | 1 + 8 files changed, 82 insertions(+), 33 deletions(-)
fs/nfs/callback_proc.c +4 -0
@@ -221,6 +221,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)list_del (&cb_info->pcl_list); clp->cl_cb_lrecall_count--; clp->cl_drain_notification[1 << cb_info->pcl_notify_bit] =NULL ; +rpc_wake_up (&clp->cl_rpcwaitq_recall);kfree (cb_info); } }
@@ -372,6 +373,7 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,list_del (&new->pcl_list); clp->cl_cb_lrecall_count--; clp->cl_drain_notification[1 << bit_num] =NULL ; +rpc_wake_up (&clp->cl_rpcwaitq_recall);spin_unlock (&clp->cl_lock);if (res ==NFS4_OK ) {if (args->cbl_recall_type ==RETURN_FILE ) {
@@ -380,6 +382,8 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp, lo =NFS_I (new->pcl_ino)->layout;spin_lock (&lo->inode->i_lock); lo->plh_block_lgets--; +if (!pnfs_layoutgets_blocked (lo,NULL )) +rpc_wake_up (&NFS_I (lo->inode)->lo_rpcwaitq_stateid);spin_unlock (&lo->inode->i_lock);put_layout_hdr (lo); }
fs/nfs/client.c +2 -0
@@ -159,6 +159,8 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_#if defined(CONFIG_NFS_V4_1 )INIT_LIST_HEAD (&clp->cl_layouts);INIT_LIST_HEAD (&clp->cl_layoutrecalls); +rpc_init_wait_queue (&clp->cl_rpcwaitq_recall, +"NFS client CB_LAYOUTRECALLS" );#endif nfs_fscache_get_client_cookie (clp);
fs/nfs/inode.c +1 -0
@@ -1461,6 +1461,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi) nfsi->delegation_state =0 ;init_rwsem (&nfsi->rwsem);rpc_init_wait_queue (&nfsi->lo_rpcwaitq,"pNFS Layoutreturn" ); +rpc_init_wait_queue (&nfsi->lo_rpcwaitq_stateid,"pNFS Layoutstateid" ); nfsi->layout =NULL ;#endif }
fs/nfs/nfs4proc.c +45 -2
80 lines not on this page · raw message
fs/nfs/pnfs.c +27 -31
136 lines not on this page · raw message
fs/nfs/pnfs.h +1 -0
@@ -218,6 +218,7 @@ enum pnfs_try_status pnfs_try_to_commit(struct nfs_write_data *,void pnfs_pageio_init_read (struct nfs_pageio_descriptor *,struct inode *,struct nfs_open_context *,struct list_head *);void pnfs_pageio_init_write (struct nfs_pageio_descriptor *,struct inode *); +bool pnfs_layoutgets_blocked (struct pnfs_layout_hdr *lo,nfs4_stateid *stateid);int pnfs_layout_process (struct nfs4_layoutget *lgp);void pnfs_free_lseg_list (struct list_head *tmp_list);void pnfs_destroy_layout (struct nfs_inode *);
include/linux/nfs_fs.h +1 -0
@@ -191,6 +191,7 @@ struct nfs_inode {/* pNFS layout information */ struct rpc_wait_queue lo_rpcwaitq; +struct rpc_wait_queue lo_rpcwaitq_stateid;struct pnfs_layout_hdr *layout;#endif /* CONFIG_NFS_V4*/ #ifdef CONFIG_NFS_FSCACHE
include/linux/nfs_fs_sb.h +1 -0
@@ -88,6 +88,7 @@ struct nfs_client {unsigned long cl_cb_lrecall_count;#define PNFS_MAX_CB_LRECALLS (64)atomic_t *cl_drain_notification[PNFS_MAX_CB_LRECALLS ]; +struct rpc_wait_queue cl_rpcwaitq_recall;struct pnfs_deviceid_cache *cl_devid_cache;/* pNFS deviceid cache */ #endif /* CONFIG_NFS_V4_1 */
Signature · 1 line
1.7.2.3
PATCH 9/9 Revert "pnfs-submit: wave2: check that partial LAYOUTGET return is ignored"
In reply to Benny HalevyThis reverts commit 0980d08ae8e980c001a31f0f29f9f1b8b5da6f94.
--- fs/nfs/pnfs.c | 11 ----------- 1 files changed, 0 insertions(+), 11 deletions(-)
fs/nfs/pnfs.c +0 -11
@@ -898,17 +898,6 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)struct nfs_client *clp =NFS_SERVER (ino)->nfs_client;int status =0 ; -/* Verify we got what we asked for. -* Note that because the xdr parsing only accepts a single -* element array, this can fail even if the server is behaving -* correctly. -*/ -if (lgp->args.range.iomode > res->range.iomode || - res->range.offset !=0 || - res->range.length !=NFS4_MAX_UINT64 ) { - status = -EINVAL ; - goto out; - }/* Inject layout blob into I/O device driver */ lseg =NFS_SERVER (ino)->pnfs_curr_ld->alloc_lseg (lo, res);if (!lseg ||IS_ERR (lseg)) {
Signature · 1 line
1.7.2.3
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Benny HalevyQuoted 42 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +16
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn {
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 10 lines
include/linux/nfs_xdr.h +7
-0+struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status; +}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;
Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
Quoted 48 lines · Trond Myklebust
Quoted 47 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +21
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status;
Quoted 8 lines
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 5 lines
include/linux/nfs_xdr.h +2
-0+}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Define "we" :)
First, the object layout driver relies on layout return for returning I/O error information. On the common, successful path, with return_on_close (that Panasas uses but others may not) I agree that CLOSE with the implicit layoutreturn semantics we discussed should do a good job too (accompanied with a respective LAYOUTCOMMIT if needed).
Then, when there's a large number of outstanding layout segments (again prob. non-files layout presuming server implementations are going to utilize whole-file layouts) proactive layoutreturn comes handy in capping the state both the client and server keep - reducing time wasted on walking long lists of items.
For CB_LAYOUTRECALL response the heart of the debate is around synchronizing with layouts in-use and in-flight layoutgets. There, having the server poll the client, who's retuning NFS4ERR_DELAY should essentially work but may be inefficient and unreliable in use cases where contention is likely enough. Eventually, when CB_LAYOUTRECALL is clear to go sending the LAYOUTRETURN or replying with CB_NOMATCHING_LAYOUT (assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid after NOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returning NFS4ERR_DELAY to CB_LAYOUTRECALL so different from implementing a barrier and doing the returns asynchronously with the CB_LAYOUTRECALL?
Benny
Quoted 2 lines
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
Quoted 58 lines · Benny Halevy
Quoted 48 lines · Trond Myklebust
Quoted 47 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +21
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status;Quoted 8 lines
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 5 lines
include/linux/nfs_xdr.h +2
-0+}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Quoted 1 line
Define "we" :)
Definition: "We who will be forced to maintain whatever is merged upstream."
Quoted 11 lines
First, the object layout driver relies on layout return for returning I/O error information. On the common, successful path, with
return_on_close(that Panasas uses but others may not) I agree that CLOSE with the implicit layoutreturn semantics we discussed should do a good job too (accompanied with a respective LAYOUTCOMMIT if needed).Then, when there's a large number of outstanding layout segments (again prob. non-files layout presuming server implementations are going to utilize whole-file layouts) proactive layoutreturn comes handy in capping the state both the client and server keep - reducing time wasted on walking long lists of items.
That assumes that you have a good policy for implementing a 'proactive layoutreturn'. What knowledge does either the client or the server have w.r.t. whether or not part of a layout is likely to be used in the near future other than 'file is open' or 'file is closed'?
What is the advantage to the client w.r.t. sending LAYOUTRETURN rather than just forgetting the layout or layout segment? If the server needs it returned, it can send a recall. If not, we are wasting processing time by sending an unnecessary RPC call.
Quoted 4 lines
For
CB_LAYOUTRECALLresponse the heart of the debate is around synchronizing with layouts in-use and in-flight layoutgets. There, having the server poll the client, who's retuningNFS4ERR_DELAYshould essentially work but may be inefficient and unreliable in use cases where contention is likely enough.
Define these use cases. Otherwise we're just talking generalities and presenting circular arguments again.
Quoted 8 lines
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?
"CB_LAYOUTRECALL request processing MUST be processed in "seqid" order at all times." (section 12.5.3).
In other words, you cannot just 'do the returns asynchronously': the CB_LAYOUTRECALL requests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
Quoted 11 lines · Trond Myklebust
Quoted 8 lines · Benny Halevy
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?
Quoted 7 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.
BTW: one consequence of the way the protocol was written is that you can't just throw out a LAYOUTRETURN for the entire file if the server just recalls a segment. Instead, you have to first return the segment, then send the LAYOUTRETURN for the entire file.
That part of the protocol is just one insane idea after another...
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Trond MyklebustQuoted 76 lines · Trond Myklebust
Quoted 58 lines · Benny Halevy
Quoted 48 lines · Trond Myklebust
Quoted 47 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +21
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status;Quoted 8 lines
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 5 lines
include/linux/nfs_xdr.h +2
-0+}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Quoted 1 line
Define "we" :)
Definition: "We who will be forced to maintain whatever is merged upstream."
Quoted 11 lines
First, the object layout driver relies on layout return for returning I/O error information. On the common, successful path, with
return_on_close(that Panasas uses but others may not) I agree that CLOSE with the implicit layoutreturn semantics we discussed should do a good job too (accompanied with a respective LAYOUTCOMMIT if needed).Then, when there's a large number of outstanding layout segments (again prob. non-files layout presuming server implementations are going to utilize whole-file layouts) proactive layoutreturn comes handy in capping the state both the client and server keep - reducing time wasted on walking long lists of items.
Quoted 5 lines
That assumes that you have a good policy for implementing a 'proactive layoutreturn'. What knowledge does either the client or the server have w.r.t. whether or not part of a layout is likely to be used in the near future other than 'file is open' or 'file is closed'?
The client can cache layout segments using a least recently used policy.
Quoted 5 lines
What is the advantage to the client w.r.t. sending LAYOUTRETURN rather than just forgetting the layout or layout segment? If the server needs it returned, it can send a recall. If not, we are wasting processing time by sending an unnecessary RPC call.
The client can know better than the server which layout segments is is more likely to reuse since the MDS does not see the layout usage activity (as it goes to the DS's).
Similarly, for CB_RECALL_ANY, the client chooses what layouts to return. Rather than dropping all the layouts it should return only the least likely to be reused.
Quoted 8 lines
Quoted 4 lines
For
CB_LAYOUTRECALLresponse the heart of the debate is around synchronizing with layouts in-use and in-flight layoutgets. There, having the server poll the client, who's retuningNFS4ERR_DELAYshould essentially work but may be inefficient and unreliable in use cases where contention is likely enough.Define these use cases. Otherwise we're just talking generalities and presenting circular arguments again.
The most common for Panasas is write sharing where multiple clients collaboratively write into a file in parallel. Although different clients write into disjoint byte ranges in the file they cross RAID stripe boundaries (which they're not aware of) and since only one client is allowed to write into a RAID stripe at a time, the layout is being recalled whenever the server detects a conflict.
Quoted 9 lines
Quoted 8 lines
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?
Quoted 9 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.
To simplify this (presumably rare) case what I had in mind is returning NFS4ERR_DELAY if there's a conflicting layout recall in progress.
Benny
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Trond MyklebustQuoted 20 lines · Trond Myklebust
Quoted 11 lines · Trond Myklebust
Quoted 8 lines · Benny Halevy
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 7 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.
Quoted 5 lines
BTW: one consequence of the way the protocol was written is that you can't just throw out a LAYOUTRETURN for the entire file if the server just recalls a segment. Instead, you have to first return the segment, then send the LAYOUTRETURN for the entire file.
It is true that the protocol requires the return of the exact recalled range but why can't the client do return the whole file before returning the recalled range?
Quoted 2 lines
That part of the protocol is just one insane idea after another...
This was done to ensure that the server and client are in-sync after a CB_LAYOUTRECALL. I agree that returning the whole layout thus resetting the layout state achieves the same goal and we should consider allowing it in the next version.
Benny
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Benny HalevyQuoted 91 lines · Benny Halevy
Quoted 76 lines · Trond Myklebust
Quoted 58 lines · Benny Halevy
Quoted 48 lines · Trond Myklebust
Quoted 47 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +21
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status;Quoted 8 lines
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 5 lines
include/linux/nfs_xdr.h +2
-0+}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Quoted 1 line
Define "we" :)
Definition: "We who will be forced to maintain whatever is merged upstream."
Quoted 11 lines
First, the object layout driver relies on layout return for returning I/O error information. On the common, successful path, with
return_on_close(that Panasas uses but others may not) I agree that CLOSE with the implicit layoutreturn semantics we discussed should do a good job too (accompanied with a respective LAYOUTCOMMIT if needed).Then, when there's a large number of outstanding layout segments (again prob. non-files layout presuming server implementations are going to utilize whole-file layouts) proactive layoutreturn comes handy in capping the state both the client and server keep - reducing time wasted on walking long lists of items.
Quoted 5 lines
That assumes that you have a good policy for implementing a 'proactive layoutreturn'. What knowledge does either the client or the server have w.r.t. whether or not part of a layout is likely to be used in the near future other than 'file is open' or 'file is closed'?
The client can cache layout segments using a least recently used policy.
Quoted 5 lines
What is the advantage to the client w.r.t. sending LAYOUTRETURN rather than just forgetting the layout or layout segment? If the server needs it returned, it can send a recall. If not, we are wasting processing time by sending an unnecessary RPC call.
Quoted 3 lines
The client can know better than the server which layout segments is is more likely to reuse since the MDS does not see the layout usage activity (as it goes to the DS's).
How does it do that? The client isn't in control here; the application is.
Sure you can track sequential writes and figure out which segment is going to be needed next, but that usually doesn't help you figure out the segment _reuse_ case.
The layout segment reuse case actually corresponds to data access patterns where it would usually make more sense for the client to cache instead of doing I/O (unless we're talking random I/O, but then the client won't know much more about layout access patterns either).
Quoted 3 lines
Similarly, for
CB_RECALL_ANY, the client chooses what layouts to return. Rather than dropping all the layouts it should return only the least likely to be reused.
That is more easily done, since both the client and server do know which files are open and which aren't. Use layout return on close to deal with this situation.
Quoted 9 lines
Quoted 8 lines
Quoted 4 lines
For
CB_LAYOUTRECALLresponse the heart of the debate is around synchronizing with layouts in-use and in-flight layoutgets. There, having the server poll the client, who's retuningNFS4ERR_DELAYshould essentially work but may be inefficient and unreliable in use cases where contention is likely enough.Define these use cases. Otherwise we're just talking generalities and presenting circular arguments again.
Quoted 6 lines
The most common for Panasas is write sharing where multiple clients collaboratively write into a file in parallel. Although different clients write into disjoint byte ranges in the file they cross RAID stripe boundaries (which they're not aware of) and since only one client is allowed to write into a RAID stripe at a time, the layout is being recalled whenever the server detects a conflict.
So basically, we're talking about the case of a shared database doing O_DIRECT without taking striping into account? Something like Oracle certainly allows you to tune its database block size to match the striping. If you are looking for performance, why wouldn't you do that?
Quoted 19 lines
Quoted 9 lines
Quoted 8 lines
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 9 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.
Quoted 2 lines
To simplify this (presumably rare) case what I had in mind is returning
NFS4ERR_DELAYif there's a conflicting layout recall in progress.
OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning NOMATCHING_LAYOUT is clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return NFS4ERR_DELAY to the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
Quoted 92 lines · Trond Myklebust
Quoted 91 lines · Benny Halevy
Quoted 76 lines · Trond Myklebust
Quoted 58 lines · Benny Halevy
Quoted 48 lines · Trond Myklebust
Quoted 47 lines · Benny Halevy
This reverts commit
19e1e5ae1ec0a3f5d997a1a5d924d482e147bea2.--- include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 0 deletions(-)include/linux/nfs4.h +1
-0@@ -557,6 +557,7 @@ enum {NFSPROC4_CLNT_RECLAIM_COMPLETE ,NFSPROC4_CLNT_LAYOUTGET ,NFSPROC4_CLNT_LAYOUTCOMMIT , +NFSPROC4_CLNT_LAYOUTRETURN ,NFSPROC4_CLNT_GETDEVICEINFO ,NFSPROC4_CLNT_PNFS_WRITE , NFSPROC4_CLNT_PNFS_COMMIT,include/linux/nfs_xdr.h +21
-0@@ -258,6 +258,29 @@ struct nfs4_layoutcommit_data {int status; }; +struct nfs4_layoutreturn_args { +__u32 reclaim; +__u32 layout_type; +__u32 return_type; +struct pnfs_layout_range range; +struct inode *inode; +struct nfs4_sequence_args seq_args; +}; + +struct nfs4_layoutreturn_res { +struct nfs4_sequence_res seq_res; +u32 lrs_present; +nfs4_stateid stateid; +}; + +struct nfs4_layoutreturn { +struct nfs4_layoutreturn_args args; +struct nfs4_layoutreturn_res res; +struct rpc_cred *cred; +struct nfs_client *clp; +int rpc_status;Quoted 8 lines
include/linux/nfs_xdr.h struct nfs4_layoutcommit_data { · line 286
Quoted 5 lines
include/linux/nfs_xdr.h +2
-0+}; +struct nfs4_getdeviceinfo_args {struct pnfs_device *pdev;struct nfs4_sequence_args seq_args;Why? We don't need or even want layoutreturn. It adds too much serialisation crap.
Quoted 1 line
Define "we" :)
Definition: "We who will be forced to maintain whatever is merged upstream."
Quoted 11 lines
First, the object layout driver relies on layout return for returning I/O error information. On the common, successful path, with
return_on_close(that Panasas uses but others may not) I agree that CLOSE with the implicit layoutreturn semantics we discussed should do a good job too (accompanied with a respective LAYOUTCOMMIT if needed).Then, when there's a large number of outstanding layout segments (again prob. non-files layout presuming server implementations are going to utilize whole-file layouts) proactive layoutreturn comes handy in capping the state both the client and server keep - reducing time wasted on walking long lists of items.
Quoted 5 lines
That assumes that you have a good policy for implementing a 'proactive layoutreturn'. What knowledge does either the client or the server have w.r.t. whether or not part of a layout is likely to be used in the near future other than 'file is open' or 'file is closed'?
The client can cache layout segments using a least recently used policy.
Quoted 5 lines
What is the advantage to the client w.r.t. sending LAYOUTRETURN rather than just forgetting the layout or layout segment? If the server needs it returned, it can send a recall. If not, we are wasting processing time by sending an unnecessary RPC call.
Quoted 6 lines
Quoted 3 lines
The client can know better than the server which layout segments is is more likely to reuse since the MDS does not see the layout usage activity (as it goes to the DS's).
How does it do that? The client isn't in control here; the application is.
Of course no one can predict reuse patterns better than the application,
Quoted 4 lines
Sure you can track sequential writes and figure out which segment is going to be needed next, but that usually doesn't help you figure out the segment _reuse_ case.
Given a finite cache on the client side it can evict old layout segments to make room for new ones, just dropping them means that the server is required to keep track of more state for the clients.
Quoted 5 lines
The layout segment reuse case actually corresponds to data access patterns where it would usually make more sense for the client to cache instead of doing I/O (unless we're talking random I/O, but then the client won't know much more about layout access patterns either).
The random I/O case is a good example where the client can return layout segments early on.
Quoted 8 lines
Quoted 3 lines
Similarly, for
CB_RECALL_ANY, the client chooses what layouts to return. Rather than dropping all the layouts it should return only the least likely to be reused.That is more easily done, since both the client and server do know which files are open and which aren't. Use layout return on close to deal with this situation.
That doesn't work for large files, with layout segments (less likely in the files-layout model), that are opened for long periods of time.
Quoted 16 lines
Quoted 9 lines
Quoted 8 lines
Quoted 4 lines
For
CB_LAYOUTRECALLresponse the heart of the debate is around synchronizing with layouts in-use and in-flight layoutgets. There, having the server poll the client, who's retuningNFS4ERR_DELAYshould essentially work but may be inefficient and unreliable in use cases where contention is likely enough.Define these use cases. Otherwise we're just talking generalities and presenting circular arguments again.
Quoted 6 lines
The most common for Panasas is write sharing where multiple clients collaboratively write into a file in parallel. Although different clients write into disjoint byte ranges in the file they cross RAID stripe boundaries (which they're not aware of) and since only one client is allowed to write into a RAID stripe at a time, the layout is being recalled whenever the server detects a conflict.
Quoted 5 lines
So basically, we're talking about the case of a shared database doing
O_DIRECTwithout taking striping into account? Something like Oracle certainly allows you to tune its database block size to match the striping. If you are looking for performance, why wouldn't you do that?
Not really, for use its number-crunching clustered applications. Their block sizes depend more on the number of clients and on the data set.
Quoted 26 lines
Quoted 19 lines
Quoted 9 lines
Quoted 8 lines
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 9 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.Quoted 2 lines
To simplify this (presumably rare) case what I had in mind is returning
NFS4ERR_DELAYif there's a conflicting layout recall in progress.OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
Quoted 5 lines
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning
NOMATCHING_LAYOUTis clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?
I agree that if the client can free the recalled layout synchronously and if it need not send a LAYOUTCOMMIT or LAYOUTRETURN (e.g. in the objects case) it can simply return NFS4ERR_NOMATCHING_LAYOUT.
Quoted 9 lines
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return
NFS4ERR_DELAYto the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?
First, the server does not have to poll the client and waste cpu and network resources on that.
Second, for the competing client, with notifications, it too does not have to poll the server and can wait on getting the notification when the layout becomes available.
Benny
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Benny HalevyHi, Benny,
Quoted 27 lines · Benny Halevy
Quoted 20 lines · Trond Myklebust
Quoted 11 lines · Trond Myklebust
Quoted 8 lines · Benny Halevy
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 7 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.Quoted 5 lines
BTW: one consequence of the way the protocol was written is that you can't just throw out a LAYOUTRETURN for the entire file if the server just recalls a segment. Instead, you have to first return the segment, then send the LAYOUTRETURN for the entire file.
Quoted 3 lines
It is true that the protocol requires the return of the exact recalled range but why can't the client do return the whole file before returning the recalled range?
Just for clarification, do you mean that after client returns more than server recalls, clients still has to do an echoing LAYOUTRETURN? It is barely overhead...
Why would server require some behavior like that?
Quoted 15 lines
Quoted 3 lines
That part of the protocol is just one insane idea after another...
This was done to ensure that the server and client are in-sync after a
CB_LAYOUTRECALL. I agree that returning the whole layout thus resetting the layout state achieves the same goal and we should consider allowing it in the next version.Benny
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 2 lines
Thanks, -Bergwolf
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Benny HalevyQuoted 11 lines · Benny Halevy
Quoted 9 lines · Trond Myklebust
OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning
NOMATCHING_LAYOUTis clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?
Quoted 3 lines
I agree that if the client can free the recalled layout synchronously and if it need not send a LAYOUTCOMMIT or LAYOUTRETURN (e.g. in the objects case) it can simply return
NFS4ERR_NOMATCHING_LAYOUT.
Objects and blocks != wave 2. We can cross that bridge when we get to it.
Quoted 10 lines
Quoted 9 lines
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return
NFS4ERR_DELAYto the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?
Quoted 2 lines
First, the server does not have to poll the client and waste cpu and network resources on that.
...but this is a ____rare____ case. If you are seeing noticeable effects on the network from this, then something is wrong. If that is ever the case, then you should be writing through the MDS anyway.
Furthermore, the MDS does need to be able to cope with NFS4ERR_DELAY anyway, so why add the extra complexity to the client?
Quoted 3 lines
Second, for the competing client, with notifications, it too does not have to poll the server and can wait on getting the notification when the layout becomes available.
There is no notification of layout availability in RFC5661. Lock notification is for byte range locks, and device id notification is for device ids. The rest is for directory notifications.
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Peng TaoQuoted 30 lines · Peng Tao
Hi, Benny,
Quoted 27 lines · Benny Halevy
Quoted 25 lines · Trond Myklebust
Quoted 11 lines · Trond Myklebust
Quoted 8 lines · Benny Halevy
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 7 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.BTW: one consequence of the way the protocol was written is that you can't just throw out a LAYOUTRETURN for the entire file if the server just recalls a segment. Instead, you have to first return the segment, then send the LAYOUTRETURN for the entire file.
Quoted 8 lines
Quoted 3 lines
It is true that the protocol requires the return of the exact recalled range but why can't the client do return the whole file before returning the recalled range?
Just for clarification, do you mean that after client returns more than server recalls, clients still has to do an echoing LAYOUTRETURN? It is barely overhead...
Why would server require some behavior like that?
The reason for that in the protocol was to provide a simple way to complete a CB_LAYOUTRECALL "meta operation" when the client returns the layout in smaller ranges than recalled. We overlooked this case of the client returning a range containing the returned range. which is easier to deal with, with further stateid related functionality we introduced after specifying how layout recall initially worked...
Benny
Quoted 18 lines
Quoted 15 lines
Quoted 3 lines
That part of the protocol is just one insane idea after another...
This was done to ensure that the server and client are in-sync after a
CB_LAYOUTRECALL. I agree that returning the whole layout thus resetting the layout state achieves the same goal and we should consider allowing it in the next version.Benny
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Trond MyklebustQuoted 12 lines · Trond Myklebust
Quoted 11 lines · Benny Halevy
Quoted 9 lines · Trond Myklebust
OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning
NOMATCHING_LAYOUTis clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?
Quoted 7 lines
Quoted 3 lines
I agree that if the client can free the recalled layout synchronously and if it need not send a LAYOUTCOMMIT or LAYOUTRETURN (e.g. in the objects case) it can simply return
NFS4ERR_NOMATCHING_LAYOUT.Objects and blocks != wave 2. We can cross that bridge when we get to it.
Right. This patchset is destined as post wave2.
Quoted 20 lines
Quoted 10 lines
Quoted 9 lines
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return
NFS4ERR_DELAYto the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?Quoted 2 lines
First, the server does not have to poll the client and waste cpu and network resources on that.
...but this is a
____rare____case. If you are seeing noticeable effects on the network from this, then something is wrong. If that is ever the case, then you should be writing through the MDS anyway.Furthermore, the MDS does need to be able to cope with
NFS4ERR_DELAYanyway, so why add the extra complexity to the client?
Quoted 8 lines
Quoted 3 lines
Second, for the competing client, with notifications, it too does not have to poll the server and can wait on getting the notification when the layout becomes available.
There is no notification of layout availability in RFC5661. Lock notification is for byte range locks, and device id notification is for device ids. The rest is for directory notifications.
Hmm, CB_RECALLABLE_OBJ_AVAIL in response to loga_signal_layout_avail...
Benny
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
Quoted 17 lines · Benny Halevy
Quoted 16 lines · Trond Myklebust
Quoted 14 lines · Benny Halevy
Quoted 9 lines · Trond Myklebust
OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning
NOMATCHING_LAYOUTis clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?I agree that if the client can free the recalled layout synchronously and if it need not send a LAYOUTCOMMIT or LAYOUTRETURN (e.g. in the objects case) it can simply return
NFS4ERR_NOMATCHING_LAYOUT.
Quoted 5 lines
Quoted 3 lines
Objects and blocks != wave 2. We can cross that bridge when we get to it.
Right. This patchset is destined as post wave2.
In that case it has a very confusing title (which certainly caught me by surprise).
Quoted 25 lines
Quoted 25 lines
Quoted 11 lines
Quoted 10 lines
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return
NFS4ERR_DELAYto the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?Quoted 2 lines
First, the server does not have to poll the client and waste cpu and network resources on that.
...but this is a
____rare____case. If you are seeing noticeable effects on the network from this, then something is wrong. If that is ever the case, then you should be writing through the MDS anyway.Furthermore, the MDS does need to be able to cope with
NFS4ERR_DELAYanyway, so why add the extra complexity to the client?Quoted 3 lines
Second, for the competing client, with notifications, it too does not have to poll the server and can wait on getting the notification when the layout becomes available.
Quoted 6 lines
Quoted 4 lines
There is no notification of layout availability in RFC5661. Lock notification is for byte range locks, and device id notification is for device ids. The rest is for directory notifications.
Hmm,
CB_RECALLABLE_OBJ_AVAILin response tologa_signal_layout_avail...
Hmm indeed. Section 12.3 states:
"CB_RECALLABLE_OBJ_AVAIL (Section 20.7) tells a client that a recallable object that it was denied (in case of pNFS, a layout denied by LAYOUTGET) due to resource exhaustion is now available."
and 18.43.3 states:
"If client sets loga_signal_layout_avail to TRUE, then it is registering with the client a "want" for a layout in the event the layout cannot be obtained due to resource exhaustion."
I can't see how that is relevant to the case where a specific LAYOUTGET requires a layout recall from another client. That's not resource exhaustion.
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Benny HalevyQuoted 40 lines · Benny Halevy
Quoted 30 lines · Peng Tao
Hi, Benny,
Quoted 27 lines · Benny Halevy
Quoted 25 lines · Trond Myklebust
Quoted 11 lines · Trond Myklebust
Quoted 8 lines · Benny Halevy
Eventually, when
CB_LAYOUTRECALLis clear to go sending the LAYOUTRETURN or replying withCB_NOMATCHING_LAYOUT(assuming no I/O error to report for pnfs-obj) should be equivalent [note: need errata to clarify the resulting stateid afterNOMATCHING_LAYOUT].
Is this the serialization "crap" you're talking about?
What makes checking the conditions for returningNFS4ERR_DELAYtoCB_LAYOUTRECALLso different from implementing a barrier and doing the returns asynchronously with theCB_LAYOUTRECALL?Quoted 7 lines
"
CB_LAYOUTRECALLrequest processing MUST be processed in "seqid" order at all times." (section 12.5.3).In other words, you cannot just 'do the returns asynchronously': the
CB_LAYOUTRECALLrequests are required by the protocol to be processed in order, which means that you must serialise those LAYOUTRETURN calls to ensure that they all happen in the order the wretched server expects.BTW: one consequence of the way the protocol was written is that you can't just throw out a LAYOUTRETURN for the entire file if the server just recalls a segment. Instead, you have to first return the segment, then send the LAYOUTRETURN for the entire file.
Quoted 8 lines
Quoted 3 lines
It is true that the protocol requires the return of the exact recalled range but why can't the client do return the whole file before returning the recalled range?
Just for clarification, do you mean that after client returns more than server recalls, clients still has to do an echoing LAYOUTRETURN? It is barely overhead...
Why would server require some behavior like that?
Quoted 6 lines
The reason for that in the protocol was to provide a simple way to complete a
CB_LAYOUTRECALL"meta operation" when the client returns the layout in smaller ranges than recalled. We overlooked this case of the client returning a range containing the returned range. which is easier to deal with, with further stateid related functionality we introduced after specifying how layout recall initially worked...
I see it. Thanks. The behavior is clearly defined in section 18.44.3. And only in the subset returning case, client has to do a matching layoutreturn to complete CB_LAYOUTRECALL.
Quoted 22 lines
Benny
Quoted 18 lines
Quoted 15 lines
Quoted 3 lines
That part of the protocol is just one insane idea after another...
This was done to ensure that the server and client are in-sync after a
CB_LAYOUTRECALL. I agree that returning the whole layout thus resetting the layout state achieves the same goal and we should consider allowing it in the next version.Benny
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 2 lines
Thanks, -Bergwolf
Re: [PATCH 1/9] Revert "pnfs-submit: wave2: remove forgotten layoutreturn struct definitions"
In reply to Trond MyklebustQuoted 67 lines · Trond Myklebust
Quoted 17 lines · Benny Halevy
Quoted 16 lines · Trond Myklebust
Quoted 14 lines · Benny Halevy
Quoted 9 lines · Trond Myklebust
OK, so why not just go the whole hog and do that for all rare cases, including the one where the server recalls a layout segment that we happen to be doing I/O to?
The case we should be optimising for is the one where the layout is recalled, and no I/O to that segment is in progress. For that case, returning OK, then doing the LAYOUTRETURN instead of just returning
NOMATCHING_LAYOUTis clearly wrong: it adds a completely unnecessary round trip to the server. Agreed?I agree that if the client can free the recalled layout synchronously and if it need not send a LAYOUTCOMMIT or LAYOUTRETURN (e.g. in the objects case) it can simply return
NFS4ERR_NOMATCHING_LAYOUT.Quoted 5 lines
Quoted 3 lines
Objects and blocks != wave 2. We can cross that bridge when we get to it.
Right. This patchset is destined as post wave2.
In that case it has a very confusing title (which certainly caught me by surprise).
Quoted 25 lines
Quoted 25 lines
Quoted 11 lines
Quoted 10 lines
As for the much rarer case of a recall of a layout that is in use, how does LAYOUTRETURN speed things up? As far as I can see, the MDS is still going to return
NFS4ERR_DELAYto the client that requested the conflicting LAYOUTGET. That client then has to resend this LAYOUTGET request, at a time when the first client may or may not have returned its layout segment. So how is LAYOUTRETURN going to make all this a fast and scalable process?Quoted 2 lines
First, the server does not have to poll the client and waste cpu and network resources on that.
...but this is a
____rare____case. If you are seeing noticeable effects on the network from this, then something is wrong. If that is ever the case, then you should be writing through the MDS anyway.Furthermore, the MDS does need to be able to cope with
NFS4ERR_DELAYanyway, so why add the extra complexity to the client?Quoted 3 lines
Second, for the competing client, with notifications, it too does not have to poll the server and can wait on getting the notification when the layout becomes available.
Quoted 6 lines
Quoted 4 lines
There is no notification of layout availability in RFC5661. Lock notification is for byte range locks, and device id notification is for device ids. The rest is for directory notifications.
Hmm,
CB_RECALLABLE_OBJ_AVAILin response tologa_signal_layout_avail...Hmm indeed. Section 12.3 states:
"
CB_RECALLABLE_OBJ_AVAIL(Section 20.7) tells a client that a recallable object that it was denied (in case of pNFS, a layout denied by LAYOUTGET) due to resource exhaustion is now available."and 18.43.3 states:
Quoted 10 lines
"If client sets
loga_signal_layout_availto TRUE, then it is registering with the client a "want" for a layout in the event the layout cannot be obtained due to resource exhaustion."I can't see how that is relevant to the case where a specific LAYOUTGET requires a layout recall from another client. That's not resource exhaustion.
Yeah, the phrasing is miserable.
It should be useful for any reason making the layout temporarily unavailable. Yet another errata entry...
Benny