Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
Due to newly-introduced 'coherency=full' O_DIRECT writes also takes the EX rw_lock like buffered writes did(rw_level == 1), it turns out messing the usage of 'level' in ocfs2_dio_end_io() up, which caused i_alloc_sem being failed to get up_read'd correctly.
This patch tries to teach ocfs2_dio_end_io to understand well on all locking stuffs by explicitly introducing a new bit for i_alloc_sem in iocb's private data, just like what we did for rw_lock.
--- fs/ocfs2/aops.c | 6 ++++-- fs/ocfs2/aops.h | 6 ++++++ fs/ocfs2/file.c | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-)
fs/ocfs2/aops.c +4 -2
@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;/* this io's submitter should not have unlocked this before we could */ BUG_ON (!ocfs2_iocb_is_rw_locked (iocb));
@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);
fs/ocfs2/aops.h +6 -0
@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \ + set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)#endif /* OCFS2_FILE_H */
fs/ocfs2/file.c +7 -2
@@ -2246,7 +2246,10 @@ relock:if (direct_io) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */ +ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);/* * Concurrent O_DIRECT writes are allowed with
@@ -2534,6 +2537,7 @@ static ssize_t ocfs2_file_aio_read(struct kiocb *iocb,if (filp->f_flags &O_DIRECT ) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; +ocfs2_iocb_set_sem_locked (iocb); ret =ocfs2_rw_lock (inode,0 );if (ret <0 ) {
@@ -2543,7 +2547,8 @@ static ssize_t ocfs2_file_aio_read(struct kiocb *iocb, rw_level =0 ;/* communicate with ocfs2_dio_end_io */ ocfs2_iocb_set_rw_locked (iocb, rw_level); - } + }else +ocfs2_iocb_clear_sem_locked (iocb);/* * We're fine letting folks race truncates and extending
Signature · 1 line
1.5.5
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
Hi Tristan,
Quoted 21 lines · Tristan Ye
Due to newly-introduced 'coherency=full'
O_DIRECTwrites also takes the EXrw_locklike buffered writesdid(rw_level == 1), it turns out messing the usage of 'level' inocfs2_dio_end_io()up, which causedi_alloc_sembeing failed to get up_read'd correctly.This patch tries to teach
ocfs2_dio_end_ioto understand well on all locking stuffs by explicitly introducing a new bit fori_alloc_semin iocb's private data, just like what we did forrw_lock.Signed-off-by Tristan Ye<[email protected]>--- fs/ocfs2/aops.c | 6 ++++-- fs/ocfs2/aops.h | 6 ++++++ fs/ocfs2/file.c | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-)fs/ocfs2/aops.c +0
-0
fs/ocfs2/aops.c static void ocfs2_dio_end_io(struct kiocb *iocb, · line 571
Quoted 5 lines
fs/ocfs2/aops.c +1
-1@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;
Is sem_locked really needed here? At least from your code below, we don't need it if we can change the sequence somehow.
Quoted 35 lines
fs/ocfs2/aops.c +3
-1/* this io's submitter should not have unlocked this before we could */ BUG_ON (!ocfs2_iocb_is_rw_locked (iocb));@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);fs/ocfs2/aops.h +6
-0@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \ + set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)#endif /* OCFS2_FILE_H */ fs/ocfs2/file.c +0
-0@@ -2246,7 +2246,10 @@ relock:if (direct_io) {
fs/ocfs2/file.c relock: · line 2255
Quoted 10 lines
fs/ocfs2/file.c +4
-1down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */ +ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);/* * Concurrent O_DIRECT writes are allowed with
Sorry, but why you clear the sem lock here? It doesn't make sense if you read the code for the first time since we have't set it before. So it looks a little bit strange.
I guess maybe we can clear it when we do up_read(&inode->i_alloc_sem)?
Or another way, why not put it with the set of rw_level.
/* communicate with ocfs2_dio_end_io */
ocfs2_iocb_set_rw_locked(iocb, rw_level);+ ocfs2_iocb_set_sem_locked(iocb, have_alloc_sem);
Regards,
Tao
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
Quoted 81 lines · Tao Ma
Hi Tristan,
Quoted 26 lines · Tristan Ye
Due to newly-introduced 'coherency=full'
O_DIRECTwrites also takes the EXrw_locklike buffered writesdid(rw_level == 1), it turns out messing the
usage of 'level' inocfs2_dio_end_io()up, which causedi_alloc_sembeing
failed to get up_read'd correctly.This patch tries to teach
ocfs2_dio_end_ioto understand well on all locking
stuffs by explicitly introducing a new bit fori_alloc_semin iocb's private
data, just like what we did forrw_lock.Signed-off-by Tristan Ye<[email protected]>--- fs/ocfs2/aops.c | 6 ++++-- fs/ocfs2/aops.h | 6 ++++++ fs/ocfs2/file.c | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-)fs/ocfs2/aops.c +0
-0fs/ocfs2/aops.c static void ocfs2_dio_end_io(struct kiocb *iocb, · line 571
Quoted 5 lines
fs/ocfs2/aops.c +1
-1@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;Is
sem_lockedreally needed here? At least from your code below, we don't need it if we can change the sequence somehow.Quoted 45 lines
fs/ocfs2/aops.c +0
-0/* this io's submitter should not have unlocked this before we could */
BUG_ON(!ocfs2_iocb_is_rw_locked(iocb));fs/ocfs2/aops.c +3
-1@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);fs/ocfs2/aops.h +6
-0@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \ + set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)#endif /* OCFS2_FILE_H */fs/ocfs2/file.c +4
-1@@ -2246,7 +2246,10 @@ relock:if (direct_io) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */ +ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);
Quoted 5 lines
fs/ocfs2/file.c relock: · line 2255
Quoted 2 lines
fs/ocfs2/file.c +0
-0/* * Concurrent O_DIRECT writes are allowed with Sorry, but why you clear the sem lock here? It doesn't make sense if you read the code for the first time since we have't set it before. So it looks a little bit strange.
Yep, an explicit clear may not needed, just for a guarantee .
Quoted 7 lines
I guess maybe we can clear it when we do
up_read(&inode->i_alloc_sem)?Or another way, why not put it with the set of
rw_level./* communicate with ocfs2_dio_end_io */ ocfs2_iocb_set_rw_locked(iocb, rw_level);+
ocfs2_iocb_set_sem_locked(iocb, have_alloc_sem);
rw_lock differs from sem a little bit, we'll be facing have rw_lock or not, besides, EX or PR locks should be identified when we do have a rw_lock.
For sem, all we need to concern is, having it or not.
Tristan.
Quoted 3 lines
Regards,
Tao
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
In reply to Tao MaQuoted 81 lines · Tao Ma
Hi Tristan,
Quoted 26 lines · Tristan Ye
Due to newly-introduced 'coherency=full'
O_DIRECTwrites also takes the EXrw_locklike buffered writesdid(rw_level == 1), it turns out messing the
usage of 'level' inocfs2_dio_end_io()up, which causedi_alloc_sembeing
failed to get up_read'd correctly.This patch tries to teach
ocfs2_dio_end_ioto understand well on all locking
stuffs by explicitly introducing a new bit fori_alloc_semin iocb's private
data, just like what we did forrw_lock.Signed-off-by Tristan Ye<[email protected]>--- fs/ocfs2/aops.c | 6 ++++-- fs/ocfs2/aops.h | 6 ++++++ fs/ocfs2/file.c | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-)fs/ocfs2/aops.c +0
-0fs/ocfs2/aops.c static void ocfs2_dio_end_io(struct kiocb *iocb, · line 571
Quoted 5 lines
fs/ocfs2/aops.c +1
-1@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;Is
sem_lockedreally needed here? At least from your code below, we don't need it if we can change the sequence somehow.Quoted 45 lines
fs/ocfs2/aops.c +0
-0/* this io's submitter should not have unlocked this before we could */
BUG_ON(!ocfs2_iocb_is_rw_locked(iocb));fs/ocfs2/aops.c +3
-1@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);fs/ocfs2/aops.h +6
-0@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \ + set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)#endif /* OCFS2_FILE_H */fs/ocfs2/file.c +4
-1@@ -2246,7 +2246,10 @@ relock:if (direct_io) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */ +ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);
Quoted 5 lines
fs/ocfs2/file.c relock: · line 2255
Quoted 2 lines
fs/ocfs2/file.c +0
-0/* * Concurrent O_DIRECT writes are allowed with Sorry, but why you clear the sem lock here? It doesn't make sense if you read the code for the first time since we have't set it before. So it looks a little bit strange.
As what I said, just for a guarantee, in case of the iocb's private data gets corrupted somehow, and the corresponding bit was incorrectly set.
Quoted 10 lines
I guess maybe we can clear it when we do
up_read(&inode->i_alloc_sem)?Or another way, why not put it with the set of
rw_level./* communicate with ocfs2_dio_end_io */ ocfs2_iocb_set_rw_locked(iocb, rw_level);+
ocfs2_iocb_set_sem_locked(iocb, have_alloc_sem);Regards,
Tao
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
In reply to Tristan YeQuoted 14 lines · Tristan Ye
fs/ocfs2/aops.c +1
-1@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;/* this io's submitter should not have unlocked this before we could */ BUG_ON (!ocfs2_iocb_is_rw_locked (iocb));
fs/ocfs2/aops.c static void ocfs2_dio_end_io(struct kiocb *iocb, · line 583
Quoted 9 lines
fs/ocfs2/aops.c +3
-1@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);
I agree with Tao, we don't need sem_locked. Just do:
if (ocfs2_iocb_is_sem_locked(iocb)) {
up_read(&inode->i_alloc_sem);
ocfs2_iocb_clear_sem_locked(iocb);
}I think it reads better.
Quoted 5 lines
fs/ocfs2/aops.h +0
-0
fs/ocfs2/aops.h static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) · line 80
Quoted 9 lines
fs/ocfs2/aops.h +6
-0@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \ + set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)
Give these bits an enum. Anonymous numbers aren't good. And
make the comment better.
Quoted 6 lines
fs/ocfs2/aops.h +0
-0#endif /* OCFS2_FILE_H */ fs/ocfs2/file.c +0
-0
fs/ocfs2/file.c relock: · line 2252
Quoted 8 lines
fs/ocfs2/file.c +4
-1@@ -2246,7 +2246,10 @@ relock:if (direct_io) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */ +ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);
I understand you're trying to avoid bad bits here, but you
should explicitly clear it at the top of the function. You know for a fact that, at the top (right before relock:), there is no sem held. Then set it as you do, and only re-clear it when you up_write().
Joel
Signature · 7 lines
"Drake! We're LEAVING!" Joel Becker Senior Development Manager Oracle E-mail: joel.becker at oracle.com Phone: (650) 506-8127
PATCH 1/1 Ocfs2: Teach 'coherency=full' O_DIRECT writes to correctly up_read i_alloc_sem.
Quoted 43 lines · Joel Becker
Quoted 14 lines · Tristan Ye
fs/ocfs2/aops.c +1
-1@@ -568,7 +568,7 @@ static void ocfs2_dio_end_io(struct kiocb *iocb, bool is_async) {struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; -int level; +int level, sem_locked;/* this io's submitter should not have unlocked this before we could */ BUG_ON (!ocfs2_iocb_is_rw_locked (iocb));fs/ocfs2/aops.c static void ocfs2_dio_end_io(struct kiocb *iocb, · line 583
Quoted 9 lines
fs/ocfs2/aops.c +3
-1@@ -576,7 +576,9 @@ static void ocfs2_dio_end_io(struct kiocb *iocb,ocfs2_iocb_clear_rw_locked (iocb); level =ocfs2_iocb_rw_locked_level (iocb); -if (!level) + sem_locked =ocfs2_iocb_is_sem_locked (iocb); +ocfs2_iocb_clear_sem_locked (iocb); +if (sem_locked)up_read (&inode->i_alloc_sem);ocfs2_rw_unlock (inode, level);I agree with Tao, we don't need sem_locked. Just do: if (ocfs2_iocb_is_sem_locked(iocb)) { up_read(&inode->i_alloc_sem); ocfs2_iocb_clear_sem_locked(iocb); }I think it reads better.
Quoted 9 lines
fs/ocfs2/aops.h +1
-0@@ -72,4 +72,10 @@ static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) clear_bit(0, (unsigned long *)&iocb->private)#define ocfs2_iocb_rw_locked_level (iocb) \ test_bit(1, (unsigned long *)&iocb->private) +#define ocfs2_iocb_set_sem_locked (iocb) \
Quoted 8 lines
fs/ocfs2/aops.h static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) · line 80
Quoted 5 lines
fs/ocfs2/aops.h +5
-0+ set_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_clear_sem_locked (iocb) \ + clear_bit(2, (unsigned long *)&iocb->private) +#define ocfs2_iocb_is_sem_locked (iocb) \ + test_bit(2, (unsigned long *)&iocb->private)Give these bits an enum. Anonymous numbers aren't good. And
make the comment better.
Sounds more reasonable;)
Quoted 12 lines
Quoted 12 lines
fs/ocfs2/aops.h +0
-0#endif /* OCFS2_FILE_H */ fs/ocfs2/file.c +1
-1@@ -2246,7 +2246,10 @@ relock:if (direct_io) {down_read (&inode->i_alloc_sem); have_alloc_sem =1 ; - } +/* communicate with ocfs2_dio_end_io */
Quoted 8 lines
fs/ocfs2/file.c relock: · line 2252
Quoted 3 lines
fs/ocfs2/file.c +3
-0+ocfs2_iocb_set_sem_locked (iocb); + }else +ocfs2_iocb_clear_sem_locked (iocb);I understand you're trying to avoid bad bits here, but you
should explicitly clear it at the top of the function. You know for a fact that, at the top (right before relock:), there is no sem held. Then set it as you do, and only re-clear it when youup_write().
Alright.
Quoted 3 lines
Joel