cifs: flush all dirty pages to server before read/write
Subsystems (2)
PATCH 1/1 cifs: flush all dirty pages to server before read/write
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")
Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)
fs/cifs/file.c +22 -0
@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;
@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;
@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;
@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;
fs/cifs/inode.c +1 -1
@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc;
Signature · 1 line
2.35.1
ping...
Quoted 105 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --2.35.1
merged into cifs-2.6.git for-next pending testing
Quoted 109 lines · Kinglong Mee
ping...
Quoted 105 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --2.35.1
Signature · 3 lines
Thanks, Steve
Got some additional review comments/questions about this patch.
In __cifs_writev isn't it likely that the write will be async and now become synchronous and also could we now have a duplicated write (flushing the write, then calling write again on that range)?
For example, the change you added
+ if (CIFS_CACHE_WRITE(CIFS_I(inode)) &&
+ inode->i_mapping && inode->i_mapping->nrpages != 0) {
+ rc = filemap_write_and_wait(inode->i_mapping);
will write synchronously all dirty pages but then proceed to call the async
rc = cifs_write_from_iter(iocb->ki_pos, ctx->len, &saved_from,
cfile, cifs_sb, &ctx->list, ctx);a few lines later. Won't this kill performance?
What was the reason for this part of the patch? Doesn't the original code end up in the same place around line 678 in mm/filemap.c
int err2 = filemap_fdatawait_range(mapping,
lstart, lend);
called from:
@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,
if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |STATX_BLOCKS)) &&
!CIFS_CACHE_READ(CIFS_I(inode)) &&
inode->i_mapping && inode->i_mapping->nrpages != 0) {- rc = filemap_fdatawait(inode->i_mapping);
+ rc = filemap_write_and_wait(inode->i_mapping);
Quoted 120 lines · Steve French
merged into cifs-2.6.git for-next pending testing
Quoted 93 lines · Kinglong Mee
ping...
Quoted 89 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;Quoted 16 lines
Quoted 14 lines
fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --Quoted 2 lines
2.35.1
--
Thanks,Steve
Signature · 3 lines
Thanks, Steve
Hi steve,
Quoted 5 lines · Steve French
Got some additional review comments/questions about this patch.
In
__cifs_writevisn't it likely that the write will be async and now become synchronous and also could we now have a duplicated write (flushing the write, then calling write again on that range)?
Yes, you are right.
But for a direct write, cifs must writes those buffer pages to server before the direct write is send to server.
Quoted 7 lines
For example, the change you added
fs/cifs/file.c +3
-0+if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping);
Quoted 6 lines
will write synchronously all dirty pages but then proceed to call the async
rc = cifs_write_from_iter(iocb->ki_pos, ctx->len, &saved_from, cfile, cifs_sb, &ctx->list, ctx);a few lines later. Won't this kill performance?
Yes, it is kill performance.
But for cache=none, i don't think the local dirty pages should left when a new direct write coming.
Quoted 15 lines
What was the reason for this part of the patch? Doesn't the original code end up in the same place around line 678 in
mm/filemap.cint err2 = filemap_fdatawait_range(mapping,
lstart, lend);called from:
(quoted diff) +0
-0@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && inode->i_mapping->nrpages != 0) {
fs/cifs/inode.c int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, · line 2443
Quoted 2 lines
fs/cifs/inode.c +1
-1- rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);
This fix is the other case when mounting cifs with nolease,
# mount -t cifs -ocache=none,nolease,nobrl,guest //cifsserverip/test /mnt/cifs # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile
fsx-linux fails too.
After rethinking of this problem, i think the core problem is
the buffer io between direct io. In cifs_getattr, it flushs dirty pages if !CIFS_CACHE_READ(CIFS_I(inode)), but when oplock is granted, it's always false, so the dirty pages are not send to server.
With write oplock granted, it's right of don't send those dirty pages to server, but the following direct ios, should be send to server directly, or gets data from the local dirty pages?
Maybe cifs should flush all dirty pages at cifs_getattr as NFS nfs_getattr does,
/* Flush out writes to the server in order to update c/mtime. */
if ((request_mask & (STATX_CTIME | STATX_MTIME)) &&
S_ISREG(inode->i_mode))
filemap_write_and_wait(inode->i_mapping);After modifing cifs_getattr as following without changing __cifs_readv/__cifs_writev, the fsx-linux test pass at -ocache=none,nolease,nobrl,guest and -ocache=none,nobrl,guest.
@@ -2438,9 +2438,8 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,
* has actual ctime, mtime and file length.
*/
if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |STATX_BLOCKS)) &&
- !CIFS_CACHE_READ(CIFS_I(inode)) &&
- inode->i_mapping && inode->i_mapping->nrpages != 0) {
- rc = filemap_fdatawait(inode->i_mapping);
+ S_ISREG(inode->i_mode)) {
+ rc = filemap_write_and_wait(inode->i_mapping);
if (rc) {
mapping_set_error(inode->i_mapping, rc);
return rc;thanks,
Kinglong Mee
Quoted 125 lines · Steve French
Quoted 120 lines
merged into cifs-2.6.git for-next pending testing
Quoted 93 lines · Kinglong Mee
ping...
Quoted 89 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;Quoted 16 lines
Quoted 14 lines
fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --Quoted 2 lines
2.35.1
--
Thanks,Steve
ronnie sahlberg
reviewerQuoted 85 lines · Kinglong Mee
Hi steve,
Quoted 5 lines · Steve French
Got some additional review comments/questions about this patch.
In
__cifs_writevisn't it likely that the write will be async and now become synchronous and also could we now have a duplicated write (flushing the write, then calling write again on that range)?Yes, you are right.
But for a direct write, cifs must writes those buffer pages to server before the direct write is send to server.Quoted 7 lines
For example, the change you added
fs/cifs/file.c +3
-0+if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping);Quoted 6 lines
will write synchronously all dirty pages but then proceed to call the async
rc = cifs_write_from_iter(iocb->ki_pos, ctx->len, &saved_from, cfile, cifs_sb, &ctx->list, ctx);a few lines later. Won't this kill performance?
Yes, it is kill performance.
But for cache=none, i don't think the local dirty pages should left when a new direct write coming.Quoted 15 lines
What was the reason for this part of the patch? Doesn't the original code end up in the same place around line 678 in
mm/filemap.cint err2 = filemap_fdatawait_range(mapping,
lstart, lend);called from:
(quoted diff) +0
-0@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && inode->i_mapping->nrpages != 0) {fs/cifs/inode.c int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, · line 2443
Quoted 2 lines
fs/cifs/inode.c +1
-1- rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);This fix is the other case when mounting cifs with nolease,
# mount -t cifs -ocache=none,nolease,nobrl,guest //cifsserverip/test /mnt/cifs # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfilefsx-linux fails too.
After rethinking of this problem, i think the core problem is the buffer io between direct io. In
cifs_getattr, it flushs dirty pages if !CIFS_CACHE_READ(CIFS_I(inode)), but when oplock is granted, it's always false, so the dirty pages are not send to server.With write oplock granted, it's right of don't send those dirty pages to server, but the following direct ios, should be send to server directly, or gets data from the local dirty pages?
Maybe cifs should flush all dirty pages at
cifs_getattras NFSnfs_getattrdoes,/* Flush out writes to the server in order to update c/mtime. */ if ((request_mask & (STATX_CTIME | STATX_MTIME)) && S_ISREG(inode->i_mode)) filemap_write_and_wait(inode->i_mapping);After modifing
cifs_getattras following without changing __cifs_readv/__cifs_writev, the fsx-linux test pass at(quoted diff) +0
-1-ocache=none,nolease,nobrl,guest and -ocache=none,nobrl,guest.@@ -2438,9 +2438,8 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, * has actual ctime, mtime and file length. */ if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&
fs/cifs/inode.c int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, · line 2446
Quoted 8 lines
fs/cifs/inode.c +2
-3- !CIFS_CACHE_READ(CIFS_I(inode)) && - inode->i_mapping && inode->i_mapping->nrpages != 0) { - rc =filemap_fdatawait (inode->i_mapping); + S_ISREG(inode->i_mode)) { + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc;
Thanks, Kinglong
So with this patch, that matches what nfs does, we have a less intrusive fix that solves the issue.
Can you send this as a separate patch to the list for review?
Quoted 129 lines
thanks,
Kinglong MeeQuoted 125 lines · Steve French
Quoted 120 lines
merged into cifs-2.6.git for-next pending testing
Quoted 93 lines · Kinglong Mee
ping...
Quoted 89 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;Quoted 16 lines
Quoted 14 lines
fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --Quoted 2 lines
2.35.1
--
Thanks,Steve
Hi steve,
What's your opinion about the new fix as nfs does?
Quoted 231 lines · ronnie sahlberg
Quoted 85 lines · Kinglong Mee
Hi steve,
Quoted 5 lines · Steve French
Got some additional review comments/questions about this patch.
In
__cifs_writevisn't it likely that the write will be async and now become synchronous and also could we now have a duplicated write (flushing the write, then calling write again on that range)?Yes, you are right.
But for a direct write, cifs must writes those buffer pages to server before the direct write is send to server.Quoted 7 lines
For example, the change you added
fs/cifs/file.c +3
-0+if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping);Quoted 6 lines
will write synchronously all dirty pages but then proceed to call the async
rc = cifs_write_from_iter(iocb->ki_pos, ctx->len, &saved_from, cfile, cifs_sb, &ctx->list, ctx);a few lines later. Won't this kill performance?
Yes, it is kill performance.
But for cache=none, i don't think the local dirty pages should left when a new direct write coming.Quoted 15 lines
What was the reason for this part of the patch? Doesn't the original code end up in the same place around line 678 in
mm/filemap.cint err2 = filemap_fdatawait_range(mapping,
lstart, lend);called from:
(quoted diff) +0
-0@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && inode->i_mapping->nrpages != 0) {fs/cifs/inode.c int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, · line 2443
Quoted 2 lines
fs/cifs/inode.c +1
-1- rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);This fix is the other case when mounting cifs with nolease,
# mount -t cifs -ocache=none,nolease,nobrl,guest //cifsserverip/test /mnt/cifs # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfilefsx-linux fails too.
After rethinking of this problem, i think the core problem is the buffer io between direct io. In
cifs_getattr, it flushs dirty pages if !CIFS_CACHE_READ(CIFS_I(inode)), but when oplock is granted, it's always false, so the dirty pages are not send to server.With write oplock granted, it's right of don't send those dirty pages to server, but the following direct ios, should be send to server directly, or gets data from the local dirty pages?
Maybe cifs should flush all dirty pages at
cifs_getattras NFSnfs_getattrdoes,/* Flush out writes to the server in order to update c/mtime. */ if ((request_mask & (STATX_CTIME | STATX_MTIME)) && S_ISREG(inode->i_mode)) filemap_write_and_wait(inode->i_mapping);After modifing
cifs_getattras following without changing __cifs_readv/__cifs_writev, the fsx-linux test pass at(quoted diff) +0
-1-ocache=none,nolease,nobrl,guest and -ocache=none,nobrl,guest.@@ -2438,9 +2438,8 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, * has actual ctime, mtime and file length. */ if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&fs/cifs/inode.c int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, · line 2446
Quoted 8 lines
fs/cifs/inode.c +2
-3- !CIFS_CACHE_READ(CIFS_I(inode)) && - inode->i_mapping && inode->i_mapping->nrpages != 0) { - rc =filemap_fdatawait (inode->i_mapping); + S_ISREG(inode->i_mode)) { + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc;Thanks, Kinglong
So with this patch, that matches what nfs does, we have a less intrusive fix that solves the issue.
Can you send this as a separate patch to the list for review?Quoted 129 lines
thanks,
Kinglong MeeQuoted 125 lines · Steve French
Quoted 120 lines
merged into cifs-2.6.git for-next pending testing
Quoted 93 lines · Kinglong Mee
ping...
Quoted 89 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;Quoted 16 lines
Quoted 14 lines
fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --Quoted 2 lines
2.35.1
--
Thanks,Steve
I was also hoping to get opinions from others this week at the FS/MM summit, but no luck so far
Quoted 237 lines · Kinglong Mee
Hi steve,
What's your opinion about the new fix as nfs does?
Quoted 231 lines · ronnie sahlberg
Quoted 85 lines · Kinglong Mee
Hi steve,
Quoted 5 lines · Steve French
Got some additional review comments/questions about this patch.
In
__cifs_writevisn't it likely that the write will be async and now become synchronous and also could we now have a duplicated write (flushing the write, then calling write again on that range)?Yes, you are right.
But for a direct write, cifs must writes those buffer pages to server before the direct write is send to server.Quoted 7 lines
For example, the change you added
(quoted diff) +3
-0+ if (CIFS_CACHE_WRITE(CIFS_I(inode)) && + inode->i_mapping && inode->i_mapping->nrpages != 0) { + rc = filemap_write_and_wait(inode->i_mapping);Quoted 6 lines
will write synchronously all dirty pages but then proceed to call the async
rc = cifs_write_from_iter(iocb->ki_pos, ctx->len, &saved_from, cfile, cifs_sb, &ctx->list, ctx);a few lines later. Won't this kill performance?
Yes, it is kill performance.
But for cache=none, i don't think the local dirty pages should left when a new direct write coming.Quoted 15 lines
What was the reason for this part of the patch? Doesn't the original code end up in the same place around line 678 in
mm/filemap.cint err2 = filemap_fdatawait_range(mapping,
lstart, lend);called from:
(quoted diff) +0
-0@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && inode->i_mapping->nrpages != 0) {Review
Quoted 2 lines
(quoted diff) +1
-1- rc = filemap_fdatawait(inode->i_mapping); + rc = filemap_write_and_wait(inode->i_mapping);This fix is the other case when mounting cifs with nolease,
# mount -t cifs -ocache=none,nolease,nobrl,guest //cifsserverip/test /mnt/cifs # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfilefsx-linux fails too.
After rethinking of this problem, i think the core problem is the buffer io between direct io. In
cifs_getattr, it flushs dirty pages if !CIFS_CACHE_READ(CIFS_I(inode)), but when oplock is granted, it's always false, so the dirty pages are not send to server.With write oplock granted, it's right of don't send those dirty pages to server, but the following direct ios, should be send to server directly, or gets data from the local dirty pages?
Maybe cifs should flush all dirty pages at
cifs_getattras NFSnfs_getattrdoes,/* Flush out writes to the server in order to update c/mtime. */ if ((request_mask & (STATX_CTIME | STATX_MTIME)) && S_ISREG(inode->i_mode)) filemap_write_and_wait(inode->i_mapping);After modifing
cifs_getattras following without changing __cifs_readv/__cifs_writev, the fsx-linux test pass at(quoted diff) +0
-1-ocache=none,nolease,nobrl,guest and -ocache=none,nobrl,guest.@@ -2438,9 +2438,8 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path, * has actual ctime, mtime and file length. */ if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE |
STATX_BLOCKS)) &&Review
Quoted 8 lines
(quoted diff) +2
-3- !CIFS_CACHE_READ(CIFS_I(inode)) && - inode->i_mapping && inode->i_mapping->nrpages != 0) { - rc = filemap_fdatawait(inode->i_mapping); + S_ISREG(inode->i_mode)) { + rc = filemap_write_and_wait(inode->i_mapping); if (rc) { mapping_set_error(inode->i_mapping, rc); return rc;Thanks, Kinglong
So with this patch, that matches what nfs does, we have a less intrusive fix that solves the issue.
Can you send this as a separate patch to the list for review?Quoted 129 lines
thanks,
Kinglong MeeQuoted 125 lines · Steve French
Quoted 120 lines
merged into cifs-2.6.git for-next pending testing
Quoted 93 lines · Kinglong Mee
ping...
Quoted 89 lines · Kinglong Mee
Testing with ltp, fsx-linux fail as,
# mount -t cifs -ocache=none,nobrl,guest //cifsserverip/test /mnt/cifs/ # dd if=/dev/zero of=/mnt/cifs//junkfile bs=8192 count=19200 conv=block # ./testcases/bin/fsx-linux -l 500000 -r 4096 -t 4096 -w 4096 -N 10000 /mnt/cifs/junkfile skipping zero size read truncating to largest ever: 0x2c000 READ BAD DATA: offset = 0x1c000, size = 0x9cc0 OFFSET GOOD BAD RANGE 0x1c000 0x09d2 000000 0x22ed operation# (mod 256) for the bad dataunknown, check HOLE and EXTEND ops LOG DUMP (10 total operations): 1: 1649662377.404010 SKIPPED (no operation) 2: 1649662377.413729 WRITE 0x3000 thru 0xdece (0xaecf bytes) HOLE 3: 1649662377.424961 WRITE 0x19000 thru 0x1b410 (0x2411 bytes) HOLE 4: 1649662377.435135 TRUNCATE UP from 0x1b411 to 0x2c000 ******WWWW 5: 1649662377.487010 MAPWRITE 0x5000 thru 0x13077 (0xe078 bytes) 6: 1649662377.495006 MAPREAD 0x8000 thru 0xe16c (0x616d bytes) 7: 1649662377.500638 MAPREAD 0x1e000 thru 0x2054d (0x254e bytes) ***RRRR*** 8: 1649662377.506165 WRITE 0x76000 thru 0x7993f (0x3940 bytes) HOLE 9: 1649662377.516674 MAPWRITE 0x1a000 thru 0x1e2fe (0x42ff bytes) ******WWWW 10: 1649662377.535312 READ 0x1c000 thru 0x25cbf (0x9cc0 bytes) ***RRRR*** Correct content saved for comparison (maybe hexdump "/mnt/cifs/junkfile" vs "/mnt/cifs/junkfile.fsxgood")Those data written at MAPWRITE is not flush to smb server, but the fallowing read gets data from the backend.
Signed-off-by Kinglong Mee <[email protected]>--- fs/cifs/file.c | 22 ++++++++++++++++++++++ fs/cifs/inode.c | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-)fs/cifs/file.c +22
-0@@ -3222,6 +3222,7 @@ static ssize_t __cifs_writev( struct kiocb *iocb, struct iov_iter *from, bool direct) { struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);ssize_t total_written =0 ;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3249,6 +3250,16 @@ static ssize_t __cifs_writev( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_writev)return -ENOSYS ;@@ -3961,6 +3972,7 @@ static ssize_t __cifs_readv( {size_t len;struct file *file = iocb->ki_filp; +struct inode *inode =file_inode (iocb->ki_filp);struct cifs_sb_info *cifs_sb;struct cifsFileInfo *cfile;struct cifs_tcon *tcon;@@ -3986,6 +3998,16 @@ static ssize_t __cifs_readv( cfile = file->private_data; tcon =tlink_tcon (cfile->tlink); +/* We need to be sure that all dirty pages are written to the server. */ +if (CIFS_CACHE_WRITE (CIFS_I (inode)) && + inode->i_mapping && inode->i_mapping->nrpages !=0 ) { + rc =filemap_write_and_wait (inode->i_mapping); +if (rc) { +mapping_set_error (inode->i_mapping, rc); +return rc; + } + } +if (!tcon->ses->server->ops->async_readv)return -ENOSYS ;Quoted 16 lines
Quoted 14 lines
fs/cifs/inode.c +1
-2@@ -2440,7 +2440,7 @@ int cifs_getattr(struct user_namespace *mnt_userns, const struct path *path,if ((request_mask & (STATX_CTIME |STATX_MTIME |STATX_SIZE |STATX_BLOCKS )) && !CIFS_CACHE_READ (CIFS_I (inode)) && inode->i_mapping && inode->i_mapping->nrpages !=0 ) { - rc =filemap_fdatawait (inode->i_mapping); + rc =filemap_write_and_wait (inode->i_mapping);if (rc) {mapping_set_error (inode->i_mapping, rc);return rc; --Quoted 2 lines
2.35.1
--
Thanks,Steve
Signature · 3 lines
Thanks, Steve