OCFS2: anti stale inode for nfs (V4)
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
changes from v3:
1, move codes that checks inode allocation bit to subfunction ocfs2_test_inode_bit().
2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.
3, add inode alloc slot validation.
--
dlmglue.c | 45 +++++++++++++++++ dlmglue.h | 2 export.c | 75 +++++++++++++++++++++++++--- inode.c | 24 ++++++++- inode.h | 1 ocfs2.h | 1 ocfs2_lockid.h | 4 + suballoc.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ suballoc.h | 2 9 files changed, 295 insertions(+), 9 deletions(-)
dlmglue.h +2 -0
@@ -115,6 +115,8 @@ void ocfs2_super_unlock(struct ocfs2_sup int ex);int ocfs2_rename_lock (struct ocfs2_super *osb);void ocfs2_rename_unlock (struct ocfs2_super *osb); +int ocfs2_nfs_sync_lock (struct ocfs2_super *osb,int ex); +void ocfs2_nfs_sync_unlock (struct ocfs2_super *osb,int ex);int ocfs2_dentry_lock (struct dentry *dentry,int ex);void ocfs2_dentry_unlock (struct dentry *dentry,int ex);int ocfs2_file_lock (struct file *file,int ex,int trylock);
export.c +67 -8
102 lines not on this page · raw message
inode.c +23 -1
@@ -112,6 +112,17 @@ void ocfs2_get_inode_flags(struct ocfs2_ oi->ip_attr |=OCFS2_DIRSYNC_FL ; } +struct inode *ocfs2_ilookup (struct super_block *sb,u64 blkno) +{ +struct ocfs2_find_inode_args args; + + args.fi_blkno = blkno; + args.fi_flags =0 ; + args.fi_ino =ino_from_blkno (sb, blkno); + args.fi_sysfile_type =0 ; + +return ilookup5 (sb, blkno, ocfs2_find_actor, &args); +}struct inode *ocfs2_iget (struct ocfs2_super *osb,u64 blkno,unsigned flags,int sysfile_type) {
@@ -949,6 +960,13 @@ void ocfs2_delete_inode(struct inode *in goto bail; } +/* Lock down the nfs_sync lock in PR mode */ + status =ocfs2_nfs_sync_lock (OCFS2_SB (inode->i_sb),0 ); +if (status <0 ) { +mlog (ML_ERROR ,"getting nfs sync lock(PR) failed %d\n" , status); +ocfs2_cleanup_delete_inode (inode,0 ); + goto bail_unblock; + }/* Lock down the inode. This gives us an up to date view of * it's metadata (for verification), and allows us to * serialize delete_inode on multiple nodes.
@@ -962,7 +980,7 @@ void ocfs2_delete_inode(struct inode *inif (status != -ENOENT )mlog_errno (status);ocfs2_cleanup_delete_inode (inode,0 ); - goto bail_unblock; + goto bail_unlock_nfs_sync; }/* Query the cluster. This will be the final decision made
@@ -1005,6 +1023,10 @@ void ocfs2_delete_inode(struct inode *in bail_unlock_inode:ocfs2_inode_unlock (inode,1 );brelse (di_bh); + +bail_unlock_nfs_sync: +ocfs2_nfs_sync_unlock (OCFS2_SB (inode->i_sb),0 ); + bail_unblock: status =sigprocmask (SIG_SETMASK , &oldset,NULL );if (status <0 )
inode.h +1 -0
@@ -124,6 +124,7 @@ void ocfs2_drop_inode(struct inode *inod/* Flags for ocfs2_iget() */ #define OCFS2_FI_FLAG_SYSFILE 0x1#define OCFS2_FI_FLAG_ORPHAN_RECOVERY 0x2 +struct inode *ocfs2_ilookup (struct super_block *sb,u64 feoff);struct inode *ocfs2_iget (struct ocfs2_super *osb,u64 feoff,unsigned flags,int sysfile_type);int ocfs2_inode_init_private (struct inode *inode);
ocfs2_lockid.h +4 -0
@@ -47,6 +47,7 @@ enum ocfs2_lock_type {OCFS2_LOCK_TYPE_OPEN ,OCFS2_LOCK_TYPE_FLOCK ,OCFS2_LOCK_TYPE_QINFO , +OCFS2_LOCK_TYPE_NFS_SYNC , OCFS2_NUM_LOCK_TYPES };
@@ -81,6 +82,9 @@ static inline char ocfs2_lock_type_char(case OCFS2_LOCK_TYPE_QINFO : c ='Q' ;break ; +case OCFS2_LOCK_TYPE_NFS_SYNC : + c ='Y' ; +break ;default : c ='\0' ; }
suballoc.c +150 -0
153 lines not on this page · raw message
suballoc.h +2 -0
@@ -186,4 +186,6 @@ int ocfs2_lock_allocators(struct inode * u32 clusters_to_add, u32 extents_to_split, struct ocfs2_alloc_context **data_ac, struct ocfs2_alloc_context **meta_ac); + +int ocfs2_test_inode_bit (struct ocfs2_super *osb,u64 blkno,int *res);#endif /* _CHAINALLOC_H_ */
ocfs2.h +1 -0
@@ -305,6 +305,7 @@ struct ocfs2_superstruct ocfs2_cluster_connection *cconn;struct ocfs2_lock_res osb_super_lockres;struct ocfs2_lock_res osb_rename_lockres; +struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;
dlmglue.c +45 -0
81 lines not on this page · raw message
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 8 lines · wengang wang
changes from v3:
1, move codes that checks inode allocation bit to subfunctionocfs2_test_inode_bit().2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.
3, add inode alloc slot validation.
Almost there!
Quoted 8 lines
export.c +8
-0+/* takes nfs_sync_lock in EX mode */ + status =ocfs2_nfs_sync_lock (osb,1 ); +if (status <0 ) { +mlog (ML_ERROR ,"getting nfs sync lock(EX) failed %d\n" , status); + goto check_err; + } + + status =ocfs2_test_inode_bit (osb, blkno, &set);
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 10 lines
export.c +10
-0+if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ + status = -EEXIST ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }
This looks very nice, but it should return -ESTALE instead
of -EEXIST; it most likely does not exist, and nfs knows how to handle that error.
export.c static struct dentry *ocfs2_get_dentry(s · line 131
Quoted 7 lines
export.c +4
-1result =d_obtain_alias (inode); -if (!IS_ERR (result)) +if (!IS_ERR (result)) { result->d_op = &ocfs2_dentry_ops; + }else { +mlog_errno ((int )result); + }
Use PTR_ERR(result), not (int)result.
Quoted 21 lines
Index:
suballoc.c
===================================================================suballoc.c +13
-0@@ -2116,3 +2116,153 @@ out:return ret; } + +/* reads(hit disk) the inode specified by blkno to get suballoc_slot +* and suballoc_bit +* */ +static int ocfs2_get_suballoc_slot_bit (struct ocfs2_super *osb,u64 blkno, +u16 *suballoc_slot,u16 *suballoc_bit) +{ +int status; +struct buffer_head *inode_bh =NULL ; +struct ocfs2_dinode *inode_fe; + +mlog_entry ("blkno: %llu\n" , blkno); +
suballoc.c out: · line 2141
Quoted 10 lines
suballoc.c +10
-0+/* dirty read disk */ + status =ocfs2_read_blocks_sync (osb, blkno,1 , &inode_bh); +if (status <0 ) + goto bail; + + inode_fe = (struct ocfs2_dinode *) inode_bh->b_data; +if (!OCFS2_IS_VALID_DINODE (inode_fe)) { + status = -EINVAL ; + goto bail; + }
Let's add an ML_ERROR here: "invalid inode %llu requested".
suballoc.c out: · line 2150
Quoted 8 lines
suballoc.c +8
-0+if (le16_to_cpu (inode_fe->i_suballoc_slot) !=OCFS2_INVALID_SLOT && + (u32 )le16_to_cpu (inode_fe->i_suballoc_slot) > osb->max_slots -1 ) { +mlog (ML_ERROR ,"inode %llu has invalid suballoc slot %u" +"this may be caused by file system crash" , blkno, + (u32 )le16_to_cpu (inode_fe->i_suballoc_slot)); + status = -EINVAL ; + goto bail; + }
Don't print "this may be caused by file system crash", as it is
much more likely that nfs is asking for a bad block. Let's jsut say "inode %llu has an invalid suballoc slot %u".
Quoted 42 lines
suballoc.c +42
-0+/* test if the bit, which is for the inode specified by blkno, in suballoc is +* set or not. +* on success, 0 is returned and *res is 1 for SET; 0 otherwise. +* when fails, errno is returned and *res is meaningless. +* MAKE SURE to hold nfs_sync_lock to revent ocfs2_delete_inode() on another +* node from accessing the same suballoc concurrently. +* */ +int ocfs2_test_inode_bit (struct ocfs2_super *osb,u64 blkno,int *res) +{ +int status; +u16 suballoc_bit =0 , suballoc_slot =0 ; +struct inode *inode_alloc_inode; +struct buffer_head *alloc_bh =NULL ; + +/* MAKE SURE nfs_sync_lock is held */ + +mlog_entry ("blkno: %llu" , blkno); + + status =ocfs2_get_suballoc_slot_bit (osb, blkno, &suballoc_slot, + &suballoc_bit); +if (status <0 ) { +mlog (ML_ERROR ,"get alloc slot and bit failed %d\n" , status); + goto bail; + } + + inode_alloc_inode = +ocfs2_get_system_file_inode (osb,INODE_ALLOC_SYSTEM_INODE , + suballoc_slot); +if (!inode_alloc_inode) { +/* the error code could be inaccurate, but we are not able to +* get the correct one. */ + status = -EEXIST ; +mlog (ML_ERROR ,"unable to get alloc inode in slot %u\n" , + (u32 )suballoc_slot); + goto bail; + } + +mutex_lock (&inode_alloc_inode->i_mutex); +/* lock down the suballoc lock it to cause other nodes flush disk and +* then release it immediately to let allocation on other nodes to go +* asap. the reason we can read the group without lock is that we just +* want to know if the bit is clear and there can't be a concurrent
suballoc.c out: · line 2254
Quoted 10 lines
suballoc.c +10
-0+* clearing action because we hold the nfs_sync_lock. sure that there +* can be a concurrent setting action then. it doesn't matter, we can +* check generation later. */ + status =ocfs2_inode_lock (inode_alloc_inode, &alloc_bh,0 ); +if (status <0 ) { +mutex_unlock (&inode_alloc_inode->i_mutex); +mlog (ML_ERROR ,"lock on alloc inode on slot %u failed %d\n" , + (u32 )suballoc_slot, status); + goto bail; + }
You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may. So move the ocfs2_test_suballoc_bit() call inside the lock here:
+ status = ocfs2_test_suballoc_bit(osb, inode_alloc_inode, alloc_bh,
+ blkno, suballoc_bit, res);
+ if (status < 0)
+ mlog(ML_ERROR, "test suballoc bit failed %d\n", status);
+ ocfs2_inode_unlock(inode_alloc_inode, 0);
+ mutex_unlock(&inode_alloc_inode->i_mutex);
Update the comment, of course. Your points about the nfs_sync_lock are correct. The only change is that we read the group under the lock.
suballoc.c out: · line 2268
Quoted 6 lines
suballoc.c +6
-0+iput (inode_alloc_inode); +brelse (alloc_bh); +bail: +mlog_exit (status); +return status; +}
Otherwise this looks good.
Joel
Signature · 9 lines
#!/bin/perl -sp0777i<X+d*lMLa^*lN%0]dsXx++lMlN/dsM0<j]dsj
$/=unpack('H*',$_);$_=`echo 16dio\U$k"SK$/SM$n\EsN0p[lN*1
lK[d2%Sa2/d0$^Ixp"|dc`;s/\W//g;$_=pack('H*',/((..)*)$/)
Joel Becker
Principal Software Developer
Oracle
E-mail: [email protected]
Phone: (650) 506-8127PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 6 lines
Quoted 6 lines
export.c +6
-0+ status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ + status = -EEXIST ;
Quoted 8 lines
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 5 lines
export.c +5
-0+ }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }This looks very nice, but it should return
-ESTALEinstead
of-EEXIST;
why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it. it most likely does not exist, and nfs knows how to handle
Quoted 7 lines
that error.
export.c static struct dentry *ocfs2_get_dentry(s · line 131
Quoted 2 lines
export.c +2
-0+mlog_errno ((int )result); + }Use
PTR_ERR(result), not (int)result.
yes:)
Quoted 6 lines
suballoc.c out: · line 2141
Quoted 4 lines
suballoc.c +4
-0+if (!OCFS2_IS_VALID_DINODE (inode_fe)) { + status = -EINVAL ; + goto bail; + }Let's add an
ML_ERRORhere: "invalid inode %llu requested".
Ok.
Quoted 4 lines
Quoted 4 lines
suballoc.c +4
-0+if (le16_to_cpu (inode_fe->i_suballoc_slot) !=OCFS2_INVALID_SLOT && + (u32 )le16_to_cpu (inode_fe->i_suballoc_slot) > osb->max_slots -1 ) { + mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u" + "this may be caused by file system crash", blkno,
Quoted 8 lines
suballoc.c out: · line 2150
Quoted 4 lines
suballoc.c +4
-0+ (u32)le16_to_cpu(inode_fe->i_suballoc_slot)); + status = -EINVAL ; + goto bail; + }Don't print "this may be caused by file system crash", as it is
much more likely that nfs is asking for a bad block. Let's jsut say "inode %llu has an invalid suballoc slot %u".
Ok.
Quoted 10 lines
Quoted 10 lines
suballoc.c +10
-0+/* lock down the suballoc lock it to cause other nodes flush disk and +* then release it immediately to let allocation on other nodes to go +* asap. the reason we can read the group without lock is that we just +* want to know if the bit is clear and there can't be a concurrent +* clearing action because we hold the nfs_sync_lock. sure that there +* can be a concurrent setting action then. it doesn't matter, we can +* check generation later. */ + status =ocfs2_inode_lock (inode_alloc_inode, &alloc_bh,0 ); +if (status <0 ) { +mutex_unlock (&inode_alloc_inode->i_mutex);
Quoted 8 lines
suballoc.c out: · line 2254
Quoted 4 lines
suballoc.c +4
-0+mlog (ML_ERROR ,"lock on alloc inode on slot %u failed %d\n" , + (u32 )suballoc_slot, status); + goto bail; + }You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.
sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.
thanks,
wengang.
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 13 lines · Wengang Wang
Quoted 5 lines
Quoted 5 lines
export.c +5
-0+ status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ Quoted 8 lines
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 6 lines
export.c +6
-0+ status = -EEXIST ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }This looks very nice, but it should return
-ESTALEinstead
of-EEXIST;
Quoted 5 lines
why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handleQuoted 1 line
that error.
EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT. But what is really happening here is nfs asks "I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".
Quoted 18 lines
Quoted 9 lines
Quoted 9 lines
suballoc.c +9
-0+/* lock down the suballoc lock it to cause other nodes flush disk and +* then release it immediately to let allocation on other nodes to go +* asap. the reason we can read the group without lock is that we just +* want to know if the bit is clear and there can't be a concurrent +* clearing action because we hold the nfs_sync_lock. sure that there +* can be a concurrent setting action then. it doesn't matter, we can +* check generation later. */ + status =ocfs2_inode_lock (inode_alloc_inode, &alloc_bh,0 ); +if (status <0 ) {Quoted 8 lines
suballoc.c out: · line 2254
Quoted 5 lines
suballoc.c +5
-0+mutex_unlock (&inode_alloc_inode->i_mutex); +mlog (ML_ERROR ,"lock on alloc inode on slot %u failed %d\n" , + (u32 )suballoc_slot, status); + goto bail; + }You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.
Quoted 8 lines
sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.
The issue is not the clear<->set transition. Remember, we're
only blocking out deletes with the nfs sync lock. Other nodes can do a create (the clear->set), but they also might do that on other group descriptors. This means the layout of groups, including their pointers, might be changing beneath us.
Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck. Putting the access of the group inside the lock is definitively correct.
Joel
Signature · 10 lines
You can use a screwdriver to screw in screws or to clean your ears, however, the latter needs real skill, determination and a lack of fear of injuring yourself. It is much the same with JavaScript. - Chris Heilmann Joel Becker Principal Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 11 lines
export.c +11
-0+ status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ + status = -EEXIST ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }This looks very nice, but it should return
-ESTALEinstead
of-EEXIST;
Quoted 8 lines
Quoted 5 lines
why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handleQuoted 1 line
that error.
EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.
Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).
+ inode_alloc_inode =
+ ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE,
+ suballoc_slot);
+ if (!inode_alloc_inode) {
+ /* the error code could be inaccurate, but we are not able to
+ * get the correct one. */
+ status = -EEXIST;
here, we don't have the accurate reason. I suggest we return PTR_ERR(xxx) instead of NULL for ocfs2_get_system_file_inode() ( _ocfs2_get_system_file_inode() actually) and change code where ocfs2_get_system_file_inode() is called.
Quoted 3 lines
But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".
yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.
Quoted 26 lines
Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
suballoc.c +9
-0+/* lock down the suballoc lock it to cause other nodes flush disk and +* then release it immediately to let allocation on other nodes to go +* asap. the reason we can read the group without lock is that we just +* want to know if the bit is clear and there can't be a concurrent +* clearing action because we hold the nfs_sync_lock. sure that there +* can be a concurrent setting action then. it doesn't matter, we can +* check generation later. */ + status =ocfs2_inode_lock (inode_alloc_inode, &alloc_bh,0 ); +if (status <0 ) {Quoted 8 lines
suballoc.c out: · line 2254
Quoted 5 lines
suballoc.c +5
-0+mutex_unlock (&inode_alloc_inode->i_mutex); +mlog (ML_ERROR ,"lock on alloc inode on slot %u failed %d\n" , + (u32 )suballoc_slot, status); + goto bail; + }You need to test the suballoc bit inside the lock. The current
code is unsafe because while the inode set value won't change in a way that matters, the structure of the alloc inode's groups may.Quoted 8 lines
sorry, i don't understand the above very well.
i know the structure of the group may change since we released the suballoc lock. while, the change can only be manipulating a bit from "clear" to "set". according to our purpose, changing from "set" to "clear" is dangerous, the reverse changing only leads us to generation check. and the gen check can tell a stale inode.
I agree as a common function, it's not good if you meant that. but if it's used only for stale inode checking, it's workable I think.
Quoted 5 lines
The issue is not the clear<->set transition. Remember, we'reonly blocking out deletes with the nfs sync lock. Other nodes can do a create (the
clear->set), but they also might do that on other group descriptors. This means the layout of groups, including their pointers, might be changing beneath us.
if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?
Quoted 2 lines
Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.
until I got clear your idea about the groups, I persist :P.
Quoted 2 lines
Putting the
access of the group inside the lock is definitively correct.
Yes, agree.
I just want a better performance when we are correct.
thanks,
wengang.
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 34 lines · Wengang Wang
Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 11 lines
export.c +11
-0+ status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ + status = -EEXIST ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }This looks very nice, but it should return
-ESTALEinstead
of-EEXIST;Quoted 7 lines
Quoted 5 lines
why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handleQuoted 1 line
that error.
EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).suballoc.c +7
-0+ inode_alloc_inode = +ocfs2_get_system_file_inode (osb,INODE_ALLOC_SYSTEM_INODE , + suballoc_slot); +if (!inode_alloc_inode) { +/* the error code could be inaccurate, but we are not able to +* get the correct one. */ + status = -EEXIST ;
Quoted 4 lines
here, we don't have the accurate reason. I suggest we return
PTR_ERR(xxx)instead of NULL forocfs2_get_system_file_inode()(_ocfs2_get_system_file_inode()actually) and change code whereocfs2_get_system_file_inode()is called.
Yeah, I noticed that we don't have an error fromocfs2_get_system_file_inode(). But we don't have any need to fix that right now. That can be a later patch if we care.
Quoted 10 lines
Quoted 3 lines
But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.
I agree with you that ENOENT is more accurate, but right now
we're playing nice with nfsd. Actually, return -ENOENT from the test function, but in ocfs2_get_dentry() change it to -ESTALE for nfsd.
Quoted 8 lines
if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?
I agree it probably doesn't affect this direct block group. At
least, unless we change the code to do some walking or whatever. But we want this function to be right no matter the behavior.
Quoted 10 lines
Quoted 2 lines
Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.until I got clear your idea about the groups, I persist :P.
Quoted 2 lines
Putting the
access of the group inside the lock is definitively correct.Yes, agree.
I just want a better performance when we are correct.
We will never sacrifice correctness for performance. You don't
access data outside of locks unless you have some really, really good reason. This won't hurt any performance - it's a standard operation we do all the time: "take lock, do something with group descriptor, drop lock". If it was a performance problem it would be a problem everywhere, and we'd solve it in a correct fashion.
Secondly, if some later code decides to use this function, they
certainly won't expect it to be doing unsafe things. And they won't be inside a very specific nfs code path.
Joel
Signature · 9 lines
Life's Little Instruction Book #69 "Whistle" Joel Becker Principal Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Quoted 34 lines · Joel Becker
Quoted 33 lines · Wengang Wang
Quoted 14 lines · Joel Becker
Quoted 13 lines · Wengang Wang
Quoted 13 lines
export.c static struct dentry *ocfs2_get_dentry(s · line 88
Quoted 11 lines
export.c +11
-0+ status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfsd gives us wrong blkno */ + status = -EEXIST ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + }This looks very nice, but it should return
-ESTALEinstead
of-EEXIST;Quoted 7 lines
Quoted 5 lines
why? I think it's an EEXIST.
if you say nfs don't know how to deal with the EEXIST, I agree to change it.
it most likely does not exist, and nfs knows how to handleQuoted 1 line
that error.
EEXIST means "the file exists", which it clearly does not.
I think you mean ENOENT.Sorry, yes I meant ENOENT.
also, the following EEXIST is wrong too(and also in some existing codes).suballoc.c +7
-0+ inode_alloc_inode = +ocfs2_get_system_file_inode (osb,INODE_ALLOC_SYSTEM_INODE , + suballoc_slot); +if (!inode_alloc_inode) { +/* the error code could be inaccurate, but we are not able to +* get the correct one. */ + status = -EEXIST ;
Quoted 9 lines
Quoted 4 lines
here, we don't have the accurate reason. I suggest we return
PTR_ERR(xxx)instead of NULL forocfs2_get_system_file_inode()(_ocfs2_get_system_file_inode()actually) and change code whereocfs2_get_system_file_inode()is called.Yeah, I noticed that we don't have an error from
ocfs2_get_system_file_inode(). But we don't have any need to fix that right now. That can be a later patch if we care.
Ok.
Quoted 10 lines
Quoted 9 lines
Quoted 3 lines
But what is really happening here is nfs asks
"I used to have file XXX, is it still there?" The correct response seems to me to be "Um, not anymore - it's stale".yes, your case is the "normal" stale case.
checking where EINVAL is returned, it's either (!OCFS2_IS_VALID_DINODE(inode_fe)) or alloc slot out of range. in the two cases, I prefer nfs passed a wrong fh(blkno) instead of a stale one, that's nfs does a buggy request.
well, returning ESTALE is also OK. I just think ENOENT is more accurate.
Quoted 4 lines
I agree with you that ENOENT is more accurate, but right now
we're playing nice with nfsd. Actually, return-ENOENTfrom the test function, but inocfs2_get_dentry()change it to-ESTALEfor nfsd.
Ok.
Quoted 9 lines
Quoted 8 lines
if my understand is correct:),
yes, when there could be a new group added to a chain. and the chain list changed(pointers). and there could be a chain relink(ocfs2_relink_block_group()).
but I don't understand why it affects the group in question. the group in question is gotten by reading the block which is numbered of inode blkno minus alloc bit. that doesn't involve any pointer. so, could you tell me more detail?
Quoted 4 lines
I agree it probably doesn't affect this direct block group. At
least, unless we change the code to do some walking or whatever. But we want this function to be right no matter the behavior.
Ok.
Quoted 9 lines
Quoted 8 lines
Quoted 2 lines
Also, it is a correctness issue. Your way may happen to work
for the stale inode checking, but that's by sheer luck.until I got clear your idea about the groups, I persist :P.
Quoted 2 lines
Putting the
access of the group inside the lock is definitively correct.Yes, agree.
I just want a better performance when we are correct.
Quoted 9 lines
We will never sacrifice correctness for performance. You don't
access data outside of locks unless you have some really, really good reason. This won't hurt any performance - it's a standard operation we do all the time: "take lock, do something with group descriptor, drop lock". If it was a performance problem it would be a problem everywhere, and we'd solve it in a correct fashion.
Secondly, if some later code decides to use this function, they
certainly won't expect it to be doing unsafe things. And they won't be inside a very specific nfs code path.
Yeah, as a common function it shouldn't be nfs specific.
thanks,
wengang.
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
In reply to wengang wangQuoted 8 lines · wengang wang
changes from v3:
1, move codes that checks inode allocation bit to subfunctionocfs2_test_inode_bit().2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.
3, add inode alloc slot validation.
One last thing, sorry I missed it.
Quoted 13 lines
suballoc.c +13
-0+static int ocfs2_test_suballoc_bit (struct ocfs2_super *osb,struct inode *suballoc, +struct buffer_head *alloc_bh,u64 blkno,u16 bit, +int *res) +{ +struct ocfs2_dinode *alloc_fe; +struct ocfs2_group_desc *group; +struct buffer_head *group_bh =NULL ; +u64 bg_blkno; +int status; + +mlog_entry ("blkno: %llu bit: %u\n" , blkno, (unsigned int )bit); + + alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data;
suballoc.c out: · line 2191
Quoted 10 lines
suballoc.c +10
-0+BUG_ON ((bit +1 ) >ocfs2_bits_per_group (&alloc_fe->id2.i_chain)); + + bg_blkno =ocfs2_which_suballoc_group (blkno, bit); + status =ocfs2_read_blocks_sync (osb, bg_blkno,1 , &group_bh); +if (status <0 ) + goto bail; + + status =ocfs2_check_group_descriptor (osb->sb, alloc_fe, group_bh); +if (status <0 ) + goto bail;
Just use ocfs2_read_group_descriptor() here. The locking code
will make sure it reads from disk if necessary.
Joel
Signature · 10 lines
One look at the From: understanding has blossomed .procmailrc grows - Alexander Viro Joel Becker Principal Software Developer Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
In reply to wengang wangHi wengang,
thanks for the work. Just one comment. See below.
Quoted 10 lines · wengang wang
changes from v3:
1, move codes that checks inode allocation bit to subfunctionocfs2_test_inode_bit().2, release the suballoc lock just after we get it. we should release it asap and doing so doesn't affect functionility.
3, add inode alloc slot validation.
Signed-off-by Wengang Wang <[email protected]>
<snip>
Quoted 53 lines
suballoc.c +53
-0+/* reads(hit disk) the inode specified by blkno to get suballoc_slot +* and suballoc_bit +* */ +static int ocfs2_get_suballoc_slot_bit (struct ocfs2_super *osb,u64 blkno, +u16 *suballoc_slot,u16 *suballoc_bit) +{ +int status; +struct buffer_head *inode_bh =NULL ; +struct ocfs2_dinode *inode_fe; + +mlog_entry ("blkno: %llu\n" , blkno); + +/* dirty read disk */ + status =ocfs2_read_blocks_sync (osb, blkno,1 , &inode_bh); +if (status <0 ) + goto bail; + + inode_fe = (struct ocfs2_dinode *) inode_bh->b_data; +if (!OCFS2_IS_VALID_DINODE (inode_fe)) { + status = -EINVAL ; + goto bail; + } + +if (le16_to_cpu (inode_fe->i_suballoc_slot) !=OCFS2_INVALID_SLOT && + (u32 )le16_to_cpu (inode_fe->i_suballoc_slot) > osb->max_slots -1 ) { +mlog (ML_ERROR ,"inode %llu has invalid suballoc slot %u" +"this may be caused by file system crash" , blkno, + (u32 )le16_to_cpu (inode_fe->i_suballoc_slot)); + status = -EINVAL ; + goto bail; + } +if (suballoc_slot) + *suballoc_slot =le16_to_cpu (inode_fe->i_suballoc_slot); +if (suballoc_bit) + *suballoc_bit=le16_to_cpu (inode_fe->i_suballoc_bit); + +bail: +brelse (inode_bh); + +mlog_exit (status); +return status; +} + +/* test whether bit is SET in allocator bitmap or not. +* on success, 0 is returned and *res is 1 for SET; 0 otherwise. +* when fails, errno is returned and *res is meaningless. +* calls this after you have cluster locked against suballoc, or you may +* get a result based on non-up2date contents +* */ +static int ocfs2_test_suballoc_bit (struct ocfs2_super *osb,struct inode *suballoc, +struct buffer_head *alloc_bh,u64 blkno,u16 bit, +int *res) +{
suballoc.c out: · line 2182
Quoted 10 lines
suballoc.c +10
-0+struct ocfs2_dinode *alloc_fe; +struct ocfs2_group_desc *group; +struct buffer_head *group_bh =NULL ; +u64 bg_blkno; +int status; + +mlog_entry ("blkno: %llu bit: %u\n" , blkno, (unsigned int )bit); + + alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data; +BUG_ON ((bit +1 ) >ocfs2_bits_per_group (&alloc_fe->id2.i_chain));
here, we shouldn't BUG_ON. It actually isn't a kernel bug. the 'bit' is got from the disk by function ocfs2_get_suballoc_slot_bit. So maybe a corrupt inode, maybe something else, but never a bug of ocfs2. ;) So just return error please.
Regards,
Tao
PATCH 1/1 OCFS2: anti stale inode for nfs (V4)
Hi,
good point! thanks!
regards,
wengang.
Quoted 7 lines
suballoc.c out: · line 2182
Quoted 2 lines
suballoc.c +2
-0+ alloc_fe = (struct ocfs2_dinode *)alloc_bh->b_data; +BUG_ON ((bit +1 ) >ocfs2_bits_per_group (&alloc_fe->id2.i_chain));here, we shouldn't
BUG_ON. It actually isn't a kernel bug. the 'bit' is got from the disk by functionocfs2_get_suballoc_slot_bit. So maybe a corrupt inode, maybe something else, but never a bug of ocfs2. ;) So just return error please.