Skip to contentSkip to conversation

CIFS: Implement cifs_strict_readv (try #3)

PATCH In review Pavel Shilovsky 2 participants 10 messages Dec 26, 2010 – Dec 27, 2010 8 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 4/6 CIFS: Implement cifs_strict_readv (try #3)

Read from the cache if we have at least Level II oplock - otherwise read from the server. Add cifs_user_readv to let the client read into iovec buffers.

---
 fs/cifs/cifsfs.c |    4 +-
 fs/cifs/cifsfs.h |    6 ++-
 fs/cifs/file.c   |  112 ++++++++++++++++++++++++++++++++++++++---------------
 3 files changed, 87 insertions(+), 35 deletions(-)
fs/cifs/cifsfs.c +2 -2
index 2b52020..0d6b005 100644
@@ -714,7 +714,7 @@ const struct file_operations cifs_file_ops = {
 const struct file_operations cifs_file_strict_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
-	.aio_read = generic_file_aio_read,
+	.aio_read = cifs_strict_readv,
 	.aio_write = cifs_file_aio_write,
 	.open = cifs_open,
 	.release = cifs_close,
@@ -770,7 +770,7 @@ const struct file_operations cifs_file_nobrl_ops = {
 const struct file_operations cifs_file_strict_nobrl_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
-	.aio_read = generic_file_aio_read,
+	.aio_read = cifs_strict_readv,
 	.aio_write = cifs_file_aio_write,
 	.open = cifs_open,
 	.release = cifs_close,
fs/cifs/cifsfs.h +5 -1
index 657738f..6ff3ca2 100644
@@ -81,7 +81,11 @@ extern int cifs_open(struct inode *inode, struct file *file);
 extern int cifs_close(struct inode *inode, struct file *file);
 extern int cifs_closedir(struct inode *inode, struct file *file);
 extern ssize_t cifs_user_read(struct file *file, char __user *read_data,
-			 size_t read_size, loff_t *poffset);
+			      size_t read_size, loff_t *poffset);
+extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+			       unsigned long nr_segs, loff_t pos);
+extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+				 unsigned long nr_segs, loff_t pos);
 extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
 			 size_t write_size, loff_t *poffset);
 extern int cifs_lock(struct file *, int, struct file_lock *);
fs/cifs/file.c +80 -32

159 lines not on this page · raw message

Signature · 1 line
1.7.3.4

A message that is not in the archive (sent off-list or not yet received): 1293357858-10006-1-git-send-email-piastryyy-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#2 lore raw

PATCH 5/6 CIFS: Implement cifs_strict_writev (try #3)

If we don't have Exclusive oplock we write a data to the server. Also set invalidate_mapping flag on the inode if we wrote something to the server. Add cifs_iovec_write to let the client write iovec buffers through CIFSSMBWrite2.

---
 fs/cifs/cifsfs.c    |   15 +++-
 fs/cifs/cifsfs.h    |    6 ++-
 fs/cifs/cifsproto.h |    2 +
 fs/cifs/file.c      |  193 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 210 insertions(+), 6 deletions(-)
fs/cifs/cifsfs.c +11 -4
index 0d6b005..76ba04e 100644
@@ -578,10 +578,17 @@ static ssize_t cifs_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
 {
 	struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode;
 	ssize_t written;
+	int rc;
 
 	written = generic_file_aio_write(iocb, iov, nr_segs, pos);
-	if (!CIFS_I(inode)->clientCanCacheAll)
-		filemap_fdatawrite(inode->i_mapping);
+
+	if (CIFS_I(inode)->clientCanCacheAll)
+		return written;
+
+	rc = filemap_fdatawrite(inode->i_mapping);
+	if (rc)
+		cFYI(1, "cifs_file_aio_write: %d rc on %p inode", rc, inode);
+
 	return written;
 }
 
@@ -715,7 +722,7 @@ const struct file_operations cifs_file_strict_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
 	.aio_read = cifs_strict_readv,
-	.aio_write = cifs_file_aio_write,
+	.aio_write = cifs_strict_writev,
 	.open = cifs_open,
 	.release = cifs_close,
 	.lock = cifs_lock,
@@ -771,7 +778,7 @@ const struct file_operations cifs_file_strict_nobrl_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
 	.aio_read = cifs_strict_readv,
-	.aio_write = cifs_file_aio_write,
+	.aio_write = cifs_strict_writev,
 	.open = cifs_open,
 	.release = cifs_close,
 	.fsync = cifs_strict_fsync,
fs/cifs/cifsfs.h +5 -1
index 6ff3ca2..2302a81 100644
@@ -87,7 +87,11 @@ extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
 extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
 				 unsigned long nr_segs, loff_t pos);
 extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
-			 size_t write_size, loff_t *poffset);
+			       size_t write_size, loff_t *poffset);
+extern ssize_t cifs_user_writev(struct kiocb *iocb, const struct iovec *iov,
+				unsigned long nr_segs, loff_t pos);
+extern ssize_t cifs_strict_writev(struct kiocb *iocb, const struct iovec *iov,
+				  unsigned long nr_segs, loff_t pos);
 extern int cifs_lock(struct file *, int, struct file_lock *);
 extern int cifs_fsync(struct file *, int);
 extern int cifs_strict_fsync(struct file *, int);
fs/cifs/cifsproto.h +2 -0
index e6d1481..e7d6c24 100644
@@ -79,6 +79,8 @@ extern int checkSMB(struct smb_hdr *smb, __u16 mid, unsigned int length);
 extern bool is_valid_oplock_break(struct smb_hdr *smb,
 				  struct TCP_Server_Info *);
 extern bool is_size_safe_to_change(struct cifsInodeInfo *, __u64 eof);
+extern void cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset,
+			    unsigned int bytes_written);
 extern struct cifsFileInfo *find_writable_file(struct cifsInodeInfo *, bool);
 extern struct cifsFileInfo *find_readable_file(struct cifsInodeInfo *, bool);
 extern unsigned int smbCalcSize(struct smb_hdr *ptr);
fs/cifs/file.c +192 -1

205 lines not on this page · raw message

Signature · 1 line
1.7.3.4

A message that is not in the archive (sent off-list or not yet received): 1293357858-10006-2-git-send-email-piastryyy-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw

Re: [PATCH 5/6] CIFS: Implement cifs_strict_writev (try #3)

Quoted 204 lines · On Sun, 26 Dec 2010 13:04:18 +0300 Pavel Shilovsky…

If we don't have Exclusive oplock we write a data to the server. Also set invalidate_mapping flag on the inode if we wrote something to the server. Add cifs_iovec_write to let the client write iovec buffers through CIFSSMBWrite2.

---
 fs/cifs/cifsfs.c    |   15 +++-
 fs/cifs/cifsfs.h    |    6 ++-
 fs/cifs/cifsproto.h |    2 +
 fs/cifs/file.c      |  193 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 210 insertions(+), 6 deletions(-)
fs/cifs/cifsfs.c +11 -4
index 0d6b005..76ba04e 100644
@@ -578,10 +578,17 @@ static ssize_t cifs_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
 {
 	struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode;
 	ssize_t written;
+	int rc;
 
 	written = generic_file_aio_write(iocb, iov, nr_segs, pos);
-	if (!CIFS_I(inode)->clientCanCacheAll)
-		filemap_fdatawrite(inode->i_mapping);
+
+	if (CIFS_I(inode)->clientCanCacheAll)
+		return written;
+
+	rc = filemap_fdatawrite(inode->i_mapping);
+	if (rc)
+		cFYI(1, "cifs_file_aio_write: %d rc on %p inode", rc, inode);
+
 	return written;
 }
 
@@ -715,7 +722,7 @@ const struct file_operations cifs_file_strict_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
 	.aio_read = cifs_strict_readv,
-	.aio_write = cifs_file_aio_write,
+	.aio_write = cifs_strict_writev,
 	.open = cifs_open,
 	.release = cifs_close,
 	.lock = cifs_lock,
@@ -771,7 +778,7 @@ const struct file_operations cifs_file_strict_nobrl_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
 	.aio_read = cifs_strict_readv,
-	.aio_write = cifs_file_aio_write,
+	.aio_write = cifs_strict_writev,
 	.open = cifs_open,
 	.release = cifs_close,
 	.fsync = cifs_strict_fsync,
fs/cifs/cifsfs.h +5 -1
index 6ff3ca2..2302a81 100644
@@ -87,7 +87,11 @@ extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
 extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
 				 unsigned long nr_segs, loff_t pos);
 extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
-			 size_t write_size, loff_t *poffset);
+			       size_t write_size, loff_t *poffset);
+extern ssize_t cifs_user_writev(struct kiocb *iocb, const struct iovec *iov,
+				unsigned long nr_segs, loff_t pos);
+extern ssize_t cifs_strict_writev(struct kiocb *iocb, const struct iovec *iov,
+				  unsigned long nr_segs, loff_t pos);
 extern int cifs_lock(struct file *, int, struct file_lock *);
 extern int cifs_fsync(struct file *, int);
 extern int cifs_strict_fsync(struct file *, int);
fs/cifs/cifsproto.h +2 -0
index e6d1481..e7d6c24 100644
@@ -79,6 +79,8 @@ extern int checkSMB(struct smb_hdr *smb, __u16 mid, unsigned int length);
 extern bool is_valid_oplock_break(struct smb_hdr *smb,
 				  struct TCP_Server_Info *);
 extern bool is_size_safe_to_change(struct cifsInodeInfo *, __u64 eof);
+extern void cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset,
+			    unsigned int bytes_written);
 extern struct cifsFileInfo *find_writable_file(struct cifsInodeInfo *, bool);
 extern struct cifsFileInfo *find_readable_file(struct cifsInodeInfo *, bool);
 extern unsigned int smbCalcSize(struct smb_hdr *ptr);
fs/cifs/file.c +103 -1
index c3bc65c..9a23a58 100644
@@ -924,7 +924,7 @@ cifs_write_timeout(struct cifsInodeInfo *cifsi, loff_t offset)
 }
 
 /* update the file size (if needed) after a write */
-static void
+void
 cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset,
 		      unsigned int bytes_written)
 {
@@ -1677,6 +1677,197 @@ int cifs_flush(struct file *file, fl_owner_t id)
 	return rc;
 }
 
+static int
+cifs_write_allocate_pages(struct page **pages, unsigned long num_pages)
+{
+	int rc = 0;
+	unsigned long i;
+
+	for (i = 0; i < num_pages; i++) {
+		pages[i] = alloc_page(__GFP_HIGHMEM);
+		if (!pages[i]) {
+			/*
+			 * save number of pages we have already allocated and
+			 * return with ENOMEM error
+			 */
+			num_pages = i;
+			rc = -ENOMEM;
+			goto error;
+		}
+	}
+
+	return rc;
+
+error:
+	for (i = 0; i < num_pages; i++)
+		put_page(pages[i]);
+	return rc;
+}
+
+static inline
+size_t get_numpages(const size_t wsize, const size_t len, size_t *cur_len)
+{
+	size_t num_pages;
+	size_t clen;
+
+	clen = min_t(const size_t, len, wsize);
+	num_pages = clen / PAGE_CACHE_SIZE;
+	if (clen % PAGE_CACHE_SIZE)
+		num_pages++;
+
+	if (cur_len)
+		*cur_len = clen;
+
+	return num_pages;
+}
+
+static ssize_t
+cifs_iovec_write(struct file *file, const struct iovec *iov,
+		 unsigned long nr_segs, loff_t *poffset)
+{
+	size_t total_written = 0, written = 0;
+	unsigned long num_pages, npages;
+	size_t copied, len, cur_len, i;
+	struct kvec *to_send;
+	struct page **pages;
+	struct iov_iter it;
+	struct inode *inode;
+	struct cifsFileInfo *open_file;
+	struct cifsTconInfo *pTcon;
+	struct cifs_sb_info *cifs_sb;
+	int xid, rc;
+
+	len = iov_length(iov, nr_segs);
+	if (!len)
+		return 0;
+
+	cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
+	num_pages = get_numpages(cifs_sb->wsize, len, &cur_len);
+
+	pages = kmalloc(sizeof(struct pages *)*num_pages, GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	to_send = kmalloc(sizeof(struct kvec)*(num_pages + 1), GFP_KERNEL);
+	if (!to_send) {
+		kfree(pages);
+		return -ENOMEM;
+	}
+
+	rc = cifs_write_allocate_pages(pages, num_pages);
+	if (rc) {
+		kfree(pages);
+		kfree(to_send);
+		return rc;
+	}
+
+	xid = GetXid();
+	open_file = file->private_data;
+	pTcon = tlink_tcon(open_file->tlink);
+	inode = file->f_path.dentry->d_inode;
+
+	iov_iter_init(&it, iov, nr_segs, len, 0);
+	npages = num_pages;
+
+	do {
+		size_t save_len = cur_len;
+		for (i = 0; i < npages; i++) {
+			copied = min_t(const size_t, cur_len, PAGE_CACHE_SIZE);
+			copied = iov_iter_copy_from_user(pages[i], &it, 0,
+							 copied);
+			cur_len -= copied;
+			iov_iter_advance(&it, copied);
+			to_send[i+1].iov_base = kmap(pages[i]);
+			to_send[i+1].iov_len = copied;

fs/cifs/file.c int cifs_flush(struct file *file, fl_owner_t id) · line 1791

Quoted 10 lines
fs/cifs/file.c +10 -0
+		}
+
+		cur_len = save_len - cur_len;
+
+		do {
+			rc = CIFSSMBWrite2(xid, pTcon, open_file->netfid,
+					   cur_len, *poffset, &written,
+					   to_send, npages, 0);
+		} while (rc == -EAGAIN);
+
		^^^^^^^^^^^^^

I think the above loop needs to be able to reclaim the filehandle if there was a reconnect event.

Quoted 72 lines
fs/cifs/file.c +72 -0
+		for (i = 0; i < npages; i++)
+			kunmap(pages[i]);
+
+		if (written) {
+			len -= written;
+			total_written += written;
+			cifs_update_eof(CIFS_I(inode), *poffset, written);
+			*poffset += written;
+		} else if (rc < 0) {
+			if (!total_written)
+				total_written = rc;
+			break;
+		}
+
+		/* get length and number of kvecs of the next write */
+		npages = get_numpages(cifs_sb->wsize, len, &cur_len);
+	} while (len > 0);
+
+	if (total_written > 0) {
+		spin_lock(&inode->i_lock);
+		if (*poffset > inode->i_size)
+			i_size_write(inode, *poffset);
+		spin_unlock(&inode->i_lock);
+	}
+
+	cifs_stats_bytes_written(pTcon, total_written);
+	mark_inode_dirty_sync(inode);
+
+	for (i = 0; i < num_pages; i++)
+		put_page(pages[i]);
+	kfree(to_send);
+	kfree(pages);
+	FreeXid(xid);
+	return total_written;
+}
+
+ssize_t cifs_user_writev(struct kiocb *iocb, const struct iovec *iov,
+			 unsigned long nr_segs, loff_t pos)
+{
+	ssize_t written;
+	struct inode *inode;
+
+	inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+	/*
+	 * BB - optimize the way when signing is disabled. We can drop this
+	 * extra memory-to-memory copying and use iovec buffers for constructing
+	 * write request.
+	 */
+
+	written = cifs_iovec_write(iocb->ki_filp, iov, nr_segs, &pos);
+	if (written > 0) {
+		CIFS_I(inode)->invalid_mapping = true;
+		iocb->ki_pos = pos;
+	}
+
+	return written;
+}
+
+ssize_t cifs_strict_writev(struct kiocb *iocb, const struct iovec *iov,
+			   unsigned long nr_segs, loff_t pos)
+{
+	struct inode *inode;
+
+	inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+	if (CIFS_I(inode)->clientCanCacheAll)
+		return generic_file_aio_write(iocb, iov, nr_segs, pos);
+
+	/*
+	 * In strict cache mode we need to write the data to the server exactly
+	 * from the pos to pos+len-1 rather than flush all affected pages

fs/cifs/file.c int cifs_flush(struct file *file, fl_owner_t id) · line 1873

Quoted 10 lines
fs/cifs/file.c +7 -0
+	 * because it may cause a error with mandatory locks on these pages but
+	 * not on the region from pos to ppos+len-1.
+	 */
+
+	return cifs_user_writev(iocb, iov, nr_segs, pos);
+}
+
 static ssize_t
 cifs_iovec_read(struct file *file, const struct iovec *iov,
 		 unsigned long nr_segs, loff_t *poffset)

...and like in the read patch, please don't add global symbols until they're needed. Make things static where possible and only make them global as it becomes necessary.

Signature · 1 line
Jeff Layton <[email protected]>

A message that is not in the archive (sent off-list or not yet received): 20101227110538.48b150c7-9yPaYZwiELC+kQycOl6kW4xkIHaj4LzF@public.gmane.org

#6 lore raw

Re: [PATCH 5/6] CIFS: Implement cifs_strict_writev (try #3)

2010/12/27 Jeff Layton <[email protected]>:

Quoted 212 lines
Quoted 209 lines · On Sun, 26 Dec 2010 13:04:18 +0300 Pavel Shilovsky…

If we don't have Exclusive oplock we write a data to the server. Also set invalidate_mapping flag on the inode if we wrote something to the server. Add cifs_iovec_write to let the client write iovec buffers through CIFSSMBWrite2.

---
 fs/cifs/cifsfs.c    |   15 +++-
 fs/cifs/cifsfs.h    |    6 ++-
 fs/cifs/cifsproto.h |    2 +
 fs/cifs/file.c      |  193 ++++++++++++++++++++++++++++++++++++++++++++++++++-

 4 files changed, 210 insertions(+), 6 deletions(-)
fs/cifs/cifsfs.c +11 -4
index 0d6b005..76ba04e 100644
@@ -578,10 +578,17 @@ static ssize_t cifs_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
  {
       struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode;
       ssize_t written;
+     int rc;
 
       written = generic_file_aio_write(iocb, iov, nr_segs, pos);
-     if (!CIFS_I(inode)->clientCanCacheAll)
-             filemap_fdatawrite(inode->i_mapping);
+
+     if (CIFS_I(inode)->clientCanCacheAll)
+             return written;
+
+     rc = filemap_fdatawrite(inode->i_mapping);
+     if (rc)
+             cFYI(1, "cifs_file_aio_write: %d rc on %p inode", rc, inode);
+
       return written;
  }
 
@@ -715,7 +722,7 @@ const struct file_operations cifs_file_strict_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
       .aio_read = cifs_strict_readv,
-     .aio_write = cifs_file_aio_write,
+     .aio_write = cifs_strict_writev,
       .open = cifs_open,
       .release = cifs_close,
       .lock = cifs_lock,
@@ -771,7 +778,7 @@ const struct file_operations cifs_file_strict_nobrl_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
       .aio_read = cifs_strict_readv,
-     .aio_write = cifs_file_aio_write,
+     .aio_write = cifs_strict_writev,
       .open = cifs_open,
       .release = cifs_close,
       .fsync = cifs_strict_fsync,
fs/cifs/cifsfs.h +5 -1
index 6ff3ca2..2302a81 100644
@@ -87,7 +87,11 @@ extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
  extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
                                unsigned long nr_segs, loff_t pos);
  extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
-                      size_t write_size, loff_t *poffset);
+                            size_t write_size, loff_t *poffset);
+extern ssize_t cifs_user_writev(struct kiocb *iocb, const struct iovec *iov,
+                             unsigned long nr_segs, loff_t pos);
+extern ssize_t cifs_strict_writev(struct kiocb *iocb, const struct iovec *iov,
+                               unsigned long nr_segs, loff_t pos);
  extern int cifs_lock(struct file *, int, struct file_lock *);
  extern int cifs_fsync(struct file *, int);
  extern int cifs_strict_fsync(struct file *, int);
fs/cifs/cifsproto.h +2 -0
index e6d1481..e7d6c24 100644
@@ -79,6 +79,8 @@ extern int checkSMB(struct smb_hdr *smb, __u16 mid, unsigned int length);
  extern bool is_valid_oplock_break(struct smb_hdr *smb,
                                 struct TCP_Server_Info *);
  extern bool is_size_safe_to_change(struct cifsInodeInfo *, __u64 eof);
+extern void cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset,
+                         unsigned int bytes_written);
  extern struct cifsFileInfo *find_writable_file(struct cifsInodeInfo *, bool);
  extern struct cifsFileInfo *find_readable_file(struct cifsInodeInfo *, bool);
  extern unsigned int smbCalcSize(struct smb_hdr *ptr);
fs/cifs/file.c +108 -1
index c3bc65c..9a23a58 100644
@@ -924,7 +924,7 @@ cifs_write_timeout(struct cifsInodeInfo *cifsi, loff_t offset)
  }
 
  /* update the file size (if needed) after a write */
-static void
+void
  cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset,
                     unsigned int bytes_written)
  {
@@ -1677,6 +1677,197 @@ int cifs_flush(struct file *file, fl_owner_t id)
       return rc;
  }
 
+static int
+cifs_write_allocate_pages(struct page **pages, unsigned long num_pages)
+{
+     int rc = 0;
+     unsigned long i;
+
+     for (i = 0; i < num_pages; i++) {
+             pages[i] = alloc_page(__GFP_HIGHMEM);
+             if (!pages[i]) {
+                     /*
+                      * save number of pages we have already allocated and
+                      * return with ENOMEM error
+                      */
+                     num_pages = i;
+                     rc = -ENOMEM;
+                     goto error;
+             }
+     }
+
+     return rc;
+
+error:
+     for (i = 0; i < num_pages; i++)
+             put_page(pages[i]);
+     return rc;
+}
+
+static inline
+size_t get_numpages(const size_t wsize, const size_t len, size_t *cur_len)
+{
+     size_t num_pages;
+     size_t clen;
+
+     clen = min_t(const size_t, len, wsize);
+     num_pages = clen / PAGE_CACHE_SIZE;
+     if (clen % PAGE_CACHE_SIZE)
+             num_pages++;
+
+     if (cur_len)
+             *cur_len = clen;
+
+     return num_pages;
+}
+
+static ssize_t
+cifs_iovec_write(struct file *file, const struct iovec *iov,
+              unsigned long nr_segs, loff_t *poffset)
+{
+     size_t total_written = 0, written = 0;
+     unsigned long num_pages, npages;
+     size_t copied, len, cur_len, i;
+     struct kvec *to_send;
+     struct page **pages;
+     struct iov_iter it;
+     struct inode *inode;
+     struct cifsFileInfo *open_file;
+     struct cifsTconInfo *pTcon;
+     struct cifs_sb_info *cifs_sb;
+     int xid, rc;
+
+     len = iov_length(iov, nr_segs);
+     if (!len)
+             return 0;
+
+     cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
+     num_pages = get_numpages(cifs_sb->wsize, len, &cur_len);
+
+     pages = kmalloc(sizeof(struct pages *)*num_pages, GFP_KERNEL);
+     if (!pages)
+             return -ENOMEM;
+
+     to_send = kmalloc(sizeof(struct kvec)*(num_pages + 1), GFP_KERNEL);
+     if (!to_send) {
+             kfree(pages);
+             return -ENOMEM;
+     }
+
+     rc = cifs_write_allocate_pages(pages, num_pages);
+     if (rc) {
+             kfree(pages);
+             kfree(to_send);
+             return rc;
+     }
+
+     xid = GetXid();
+     open_file = file->private_data;
+     pTcon = tlink_tcon(open_file->tlink);
+     inode = file->f_path.dentry->d_inode;
+
+     iov_iter_init(&it, iov, nr_segs, len, 0);
+     npages = num_pages;
+
+     do {
+             size_t save_len = cur_len;
+             for (i = 0; i < npages; i++) {
+                     copied = min_t(const size_t, cur_len, PAGE_CACHE_SIZE);
+                     copied = iov_iter_copy_from_user(pages[i], &it, 0,
+                                                      copied);
+                     cur_len -= copied;
+                     iov_iter_advance(&it, copied);
+                     to_send[i+1].iov_base = kmap(pages[i]);
+                     to_send[i+1].iov_len = copied;
+             }
+
+             cur_len = save_len - cur_len;
+
+             do {
Quoted 8 lines

fs/cifs/file.c int cifs_flush(struct file *file, fl_owner_t id) · line 1791

Quoted 5 lines
fs/cifs/file.c +5 -0
+                     rc = CIFSSMBWrite2(xid, pTcon, open_file->netfid,
+                                        cur_len, *poffset, &written,
+                                        to_send, npages, 0);
+             } while (rc == -EAGAIN);
+

               ^^^^^^^^^^^^^
I think the above loop needs to be able to reclaim the filehandle if there was a reconnect event.

May I should add both checks: file->private_data == NULL and open_file->invalidHandle like we have them here -http://tomoyo.sourceforge.jp/cgi-bin/lxr/source/fs/cifs/file.c#L963 ?

Quoted 92 lines
Quoted 80 lines
fs/cifs/file.c +79 -0
+             for (i = 0; i < npages; i++)
+                     kunmap(pages[i]);
+
+             if (written) {
+                     len -= written;
+                     total_written += written;
+                     cifs_update_eof(CIFS_I(inode), *poffset, written);
+                     *poffset += written;
+             } else if (rc < 0) {
+                     if (!total_written)
+                             total_written = rc;
+                     break;
+             }
+
+             /* get length and number of kvecs of the next write */
+             npages = get_numpages(cifs_sb->wsize, len, &cur_len);
+     } while (len > 0);
+
+     if (total_written > 0) {
+             spin_lock(&inode->i_lock);
+             if (*poffset > inode->i_size)
+                     i_size_write(inode, *poffset);
+             spin_unlock(&inode->i_lock);
+     }
+
+     cifs_stats_bytes_written(pTcon, total_written);
+     mark_inode_dirty_sync(inode);
+
+     for (i = 0; i < num_pages; i++)
+             put_page(pages[i]);
+     kfree(to_send);
+     kfree(pages);
+     FreeXid(xid);
+     return total_written;
+}
+
+ssize_t cifs_user_writev(struct kiocb *iocb, const struct iovec *iov,
+                      unsigned long nr_segs, loff_t pos)
+{
+     ssize_t written;
+     struct inode *inode;
+
+     inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+     /*
+      * BB - optimize the way when signing is disabled. We can drop this
+      * extra memory-to-memory copying and use iovec buffers for constructing
+      * write request.
+      */
+
+     written = cifs_iovec_write(iocb->ki_filp, iov, nr_segs, &pos);
+     if (written > 0) {
+             CIFS_I(inode)->invalid_mapping = true;
+             iocb->ki_pos = pos;
+     }
+
+     return written;
+}
+
+ssize_t cifs_strict_writev(struct kiocb *iocb, const struct iovec *iov,
+                        unsigned long nr_segs, loff_t pos)
+{
+     struct inode *inode;
+
+     inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+     if (CIFS_I(inode)->clientCanCacheAll)
+             return generic_file_aio_write(iocb, iov, nr_segs, pos);
+
+     /*
+      * In strict cache mode we need to write the data to the server exactly
+      * from the pos to pos+len-1 rather than flush all affected pages
+      * because it may cause a error with mandatory locks on these pages but
+      * not on the region from pos to ppos+len-1.
+      */
+
+     return cifs_user_writev(iocb, iov, nr_segs, pos);
+}
+
Quoted 3 lines

 static ssize_t
 cifs_iovec_read(struct file *file, const struct iovec *iov, unsigned long nr_segs, loff_t *poffset)

...and like in the read patch, please don't add global symbols until they're needed. Make things static where possible and only make them global as it becomes necessary.

--
Jeff Layton <[email protected]>

Signature · 2 lines
Best regards,
Pavel Shilovsky.

A message that is not in the archive (sent off-list or not yet received): AANLkTi=+Nw+Spp6pmDJbqn5sBt0TKp8g=wr2BTHHdSYu-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#8 lore raw

Re: [PATCH 5/6] CIFS: Implement cifs_strict_writev (try #3)

Quoted 19 lines · On Mon, 27 Dec 2010 19:20:05 +0300 Pavel Shilovsky…

2010/12/27 Jeff Layton <[email protected]>:

Quoted 18 lines
Quoted 15 lines · On Sun, 26 Dec 2010 13:04:18 +0300 Pavel Shilovsky…

If we don't have Exclusive oplock we write a data to the server. Also set invalidate_mapping flag on the inode if we wrote something to the server. Add cifs_iovec_write to let the client write iovec buffers through CIFSSMBWrite2.

---
 fs/cifs/cifsfs.c    |   15 +++-
 fs/cifs/cifsfs.h    |    6 ++-
 fs/cifs/cifsproto.h |    2 +
 fs/cifs/file.c      |  193 ++++++++++++++++++++++++++++++++++++++++++++++++++-

 4 files changed, 210 insertions(+), 6 deletions(-)
fs/cifs/cifsfs.c +0 -0
index 0d6b005..76ba04e 100644

[...]

Quoted 11 lines
Quoted 10 lines

fs/cifs/cifsfs.c

Quoted 7 lines
fs/cifs/cifsfs.c +7 -0
+
+             do {
+                     rc = CIFSSMBWrite2(xid, pTcon, open_file->netfid,
+                                        cur_len, *poffset, &written,
+                                        to_send, npages, 0);
+             } while (rc == -EAGAIN);
+

               ^^^^^^^^^^^^^
I think the above loop needs to be able to reclaim the filehandle if there was a reconnect event.

Quoted 4 lines

May I should add both checks: file->private_data == NULL and open_file->invalidHandle like we have them here -http://tomoyo.sourceforge.jp/cgi-bin/lxr/source/fs/cifs/file.c#L963 ?

A private_data of NULL on an open file is now a bug. Directories can currently have a NULL private_data, but that shouldn't be a problem here. I'd just check for invalidHandle. CIFS also has a ton of unneeded NULL pointer checks so it's not always worthwhile to use existing code as a guideline.

Signature · 1 line
Jeff Layton <[email protected]>

A message that is not in the archive (sent off-list or not yet received): 20101227113556.56f1b3e8-9yPaYZwiELC+kQycOl6kW4xkIHaj4LzF@public.gmane.org

#10 lore raw

Re: [PATCH 5/6] CIFS: Implement cifs_strict_writev (try #3)

2010/12/27 Jeff Layton <[email protected]>:

Quoted 5 lines

A private_data of NULL on an open file is now a bug. Directories can currently have a NULL private_data, but that shouldn't be a problem here. I'd just check for invalidHandle. CIFS also has a ton of unneeded NULL pointer checks so it's not always worthwhile to use existing code as a guideline.

Ok, thanks for this info. In this case I'll remove private_data == NULL check from cifs_iovec_readv functions from the read patch too.

Signature · 2 lines
Best regards,
Pavel Shilovsky.
#11 lore raw
In reply to a message not in the archive
Quoted 38 lines · On Sun, 26 Dec 2010 13:04:17 +0300 Pavel Shilovsky…

Read from the cache if we have at least Level II oplock - otherwise read from the server. Add cifs_user_readv to let the client read into iovec buffers.

---
 fs/cifs/cifsfs.c |    4 +-
 fs/cifs/cifsfs.h |    6 ++-
 fs/cifs/file.c   |  112 ++++++++++++++++++++++++++++++++++++++---------------
 3 files changed, 87 insertions(+), 35 deletions(-)
fs/cifs/cifsfs.c +2 -2
index 2b52020..0d6b005 100644
@@ -714,7 +714,7 @@ const struct file_operations cifs_file_ops = {
 const struct file_operations cifs_file_strict_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
-	.aio_read = generic_file_aio_read,
+	.aio_read = cifs_strict_readv,
 	.aio_write = cifs_file_aio_write,
 	.open = cifs_open,
 	.release = cifs_close,
@@ -770,7 +770,7 @@ const struct file_operations cifs_file_nobrl_ops = {
 const struct file_operations cifs_file_strict_nobrl_ops = {
 	.read = do_sync_read,
 	.write = do_sync_write,
-	.aio_read = generic_file_aio_read,
+	.aio_read = cifs_strict_readv,
 	.aio_write = cifs_file_aio_write,
 	.open = cifs_open,
 	.release = cifs_close,
fs/cifs/cifsfs.h +0 -0
index 657738f..6ff3ca2 100644

fs/cifs/cifsfs.h extern int cifs_open(struct inode *inode, struct file *file); · line 86

Quoted 7 lines
fs/cifs/cifsfs.h +3 -1
@@ -81,7 +81,11 @@ extern int cifs_open(struct inode *inode, struct file *file);
 extern int cifs_close(struct inode *inode, struct file *file);
 extern int cifs_closedir(struct inode *inode, struct file *file);
 extern ssize_t cifs_user_read(struct file *file, char __user *read_data,
-			 size_t read_size, loff_t *poffset);
+			      size_t read_size, loff_t *poffset);
+extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+			       unsigned long nr_segs, loff_t pos);
		^^^^
	this can be static. There are no callers of cifs_user_readv.
Quoted 86 lines
fs/cifs/cifsfs.h +2 -0
+extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+				 unsigned long nr_segs, loff_t pos);
 extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
 			 size_t write_size, loff_t *poffset);
 extern int cifs_lock(struct file *, int, struct file_lock *);
fs/cifs/file.c +21 -26
index d48723c..c3bc65c 100644
@@ -1677,20 +1677,27 @@ int cifs_flush(struct file *file, fl_owner_t id)
 	return rc;
 }
 
-ssize_t cifs_user_read(struct file *file, char __user *read_data,
-	size_t read_size, loff_t *poffset)
+static ssize_t
+cifs_iovec_read(struct file *file, const struct iovec *iov,
+		 unsigned long nr_segs, loff_t *poffset)
 {
-	int rc = -EACCES;
-	unsigned int bytes_read = 0;
-	unsigned int total_read = 0;
-	unsigned int current_read_size;
+	int rc;
+	int xid;
+	unsigned int total_read, bytes_read = 0;
+	size_t len, cur_len;
+	int iov_offset = 0;
 	struct cifs_sb_info *cifs_sb;
 	struct cifsTconInfo *pTcon;
-	int xid;
 	struct cifsFileInfo *open_file;
-	char *smb_read_data;
-	char __user *current_offset;
 	struct smb_com_read_rsp *pSMBr;
+	char *read_data;
+
+	if (!nr_segs)
+		return 0;
+
+	len = iov_length(iov, nr_segs);
+	if (!len)
+		return 0;
 
 	xid = GetXid();
 	cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
@@ -1700,19 +1707,18 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
 		FreeXid(xid);
 		return rc;
 	}
+
 	open_file = file->private_data;
 	pTcon = tlink_tcon(open_file->tlink);
 
 	if ((file->f_flags & O_ACCMODE) == O_WRONLY)
 		cFYI(1, "attempting read on write only file instance");
 
-	for (total_read = 0, current_offset = read_data;
-	     read_size > total_read;
-	     total_read += bytes_read, current_offset += bytes_read) {
-		current_read_size = min_t(const int, read_size - total_read,
-					  cifs_sb->rsize);
+	for (total_read = 0; total_read < len; total_read += bytes_read) {
+		cur_len = min_t(const size_t, len - total_read, cifs_sb->rsize);
 		rc = -EAGAIN;
-		smb_read_data = NULL;
+		read_data = NULL;
+
 		while (rc == -EAGAIN) {
 			int buf_type = CIFS_NO_BUFFER;
 			if (open_file->invalidHandle) {
@@ -1720,27 +1726,25 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
 				if (rc != 0)
 					break;
 			}
-			rc = CIFSSMBRead(xid, pTcon,
-					 open_file->netfid,
-					 current_read_size, *poffset,
-					 &bytes_read, &smb_read_data,
-					 &buf_type);
-			pSMBr = (struct smb_com_read_rsp *)smb_read_data;
-			if (smb_read_data) {
-				if (copy_to_user(current_offset,
-						smb_read_data +
-						4 /* RFC1001 length field */ +
-						le16_to_cpu(pSMBr->DataOffset),

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1737

Quoted 10 lines
fs/cifs/file.c +9 -1
-						bytes_read))
+			rc = CIFSSMBRead(xid, pTcon, open_file->netfid,
+					 cur_len, *poffset, &bytes_read,
+					 &read_data, &buf_type);
+			pSMBr = (struct smb_com_read_rsp *)read_data;
+			if (read_data) {
+				char *data_offset = read_data + 4 +
+						le16_to_cpu(pSMBr->DataOffset);
+				if (memcpy_toiovecend(iov, data_offset,
+						      iov_offset, bytes_read))
					^^^^^
			Much better. It's always better to use existing
			infrastructure.
Quoted 38 lines
fs/cifs/file.c +18 -4
 					rc = -EFAULT;
-
 				if (buf_type == CIFS_SMALL_BUFFER)
-					cifs_small_buf_release(smb_read_data);
+					cifs_small_buf_release(read_data);
 				else if (buf_type == CIFS_LARGE_BUFFER)
-					cifs_buf_release(smb_read_data);
-				smb_read_data = NULL;
+					cifs_buf_release(read_data);
+				read_data = NULL;
+				iov_offset += bytes_read;
 			}
 		}
+
 		if (rc || (bytes_read == 0)) {
 			if (total_read) {
 				break;
@@ -1753,13 +1757,57 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
 			*poffset += bytes_read;
 		}
 	}
+
 	FreeXid(xid);
 	return total_read;
 }
 
+ssize_t cifs_user_read(struct file *file, char __user *read_data,
+		       size_t read_size, loff_t *poffset)
+{
+	struct iovec iov;
+	iov.iov_base = read_data;
+	iov.iov_len = read_size;
+
+	return cifs_iovec_read(file, &iov, 1, poffset);
+}
+
+ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+			unsigned long nr_segs, loff_t pos)

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1786

Quoted 10 lines
fs/cifs/file.c +10 -0
+{
+	ssize_t read;
+
+	read = cifs_iovec_read(iocb->ki_filp, iov, nr_segs, &pos);
+	if (read > 0)
+		iocb->ki_pos = pos;
+
+	return read;
+}
+

Again, I'd make the above static. If you do that, the compiler will likely just inline the function into the only caller below.

Quoted 18 lines
fs/cifs/file.c +18 -0
+ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+			  unsigned long nr_segs, loff_t pos)
+{
+	struct inode *inode;
+
+	inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+	if (CIFS_I(inode)->clientCanCacheRead)
+		return generic_file_aio_read(iocb, iov, nr_segs, pos);
+
+	/*
+	 * In strict cache mode we need to read from the server all the time
+	 * if we don't have level II oplock because the server can delay mtime
+	 * change - so we can't make a decision about inode invalidating.
+	 * And we can also fail with pagereading if there are mandatory locks
+	 * on pages affected by this read but not on the region from pos to
+	 * pos+len-1.
+	 */

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1813

Quoted 10 lines
fs/cifs/file.c +4 -1
+
+	return cifs_user_readv(iocb, iov, nr_segs, pos);
+}
 
 static ssize_t cifs_read(struct file *file, char *read_data, size_t read_size,
-	loff_t *poffset)
+			 loff_t *poffset)
 {
 	int rc = -EACCES;
 	unsigned int bytes_read = 0;

Nice work. Other than making that function static, this looks good to me, so you can add my Reviewed-by once that's been fixed...

Signature · 1 line
Jeff Layton <[email protected]>

A message that is not in the archive (sent off-list or not yet received): 20101227103556.20f329d1-9yPaYZwiELC+kQycOl6kW4xkIHaj4LzF@public.gmane.org

#13 lore raw

2010/12/27 Jeff Layton <[email protected]>:

Quoted 236 lines
Quoted 38 lines · On Sun, 26 Dec 2010 13:04:17 +0300 Pavel Shilovsky…

Read from the cache if we have at least Level II oplock - otherwise read from the server. Add cifs_user_readv to let the client read into iovec buffers.

---
 fs/cifs/cifsfs.c |    4 +-
 fs/cifs/cifsfs.h |    6 ++-
 fs/cifs/file.c   |  112 ++++++++++++++++++++++++++++++++++++++---------------

 3 files changed, 87 insertions(+), 35 deletions(-)
fs/cifs/cifsfs.c +2 -2
index 2b52020..0d6b005 100644
@@ -714,7 +714,7 @@ const struct file_operations cifs_file_ops = {
  const struct file_operations cifs_file_strict_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
@@ -770,7 +770,7 @@ const struct file_operations cifs_file_nobrl_ops = {
  const struct file_operations cifs_file_strict_nobrl_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
fs/cifs/cifsfs.h +0 -0
index 657738f..6ff3ca2 100644

fs/cifs/cifsfs.h extern int cifs_open(struct inode *inode, struct file *file); · line 86

Quoted 7 lines
fs/cifs/cifsfs.h +3 -1
@@ -81,7 +81,11 @@ extern int cifs_open(struct inode *inode, struct file *file);
  extern int cifs_close(struct inode *inode, struct file *file);
  extern int cifs_closedir(struct inode *inode, struct file *file);
  extern ssize_t cifs_user_read(struct file *file, char __user *read_data,
-                      size_t read_size, loff_t *poffset);
+                           size_t read_size, loff_t *poffset);
+extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                            unsigned long nr_segs, loff_t pos);

               ^^^^
       this can be static. There are no callers of cifs_user_readv.

Quoted 86 lines
fs/cifs/cifsfs.h +2 -0
+extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                              unsigned long nr_segs, loff_t pos);
  extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
                        size_t write_size, loff_t *poffset);
  extern int cifs_lock(struct file *, int, struct file_lock *);
fs/cifs/file.c +21 -26
index d48723c..c3bc65c 100644
@@ -1677,20 +1677,27 @@ int cifs_flush(struct file *file, fl_owner_t id)
       return rc;
  }
 
-ssize_t cifs_user_read(struct file *file, char __user *read_data,
-     size_t read_size, loff_t *poffset)
+static ssize_t
+cifs_iovec_read(struct file *file, const struct iovec *iov,
+              unsigned long nr_segs, loff_t *poffset)
  {
-     int rc = -EACCES;
-     unsigned int bytes_read = 0;
-     unsigned int total_read = 0;
-     unsigned int current_read_size;
+     int rc;
+     int xid;
+     unsigned int total_read, bytes_read = 0;
+     size_t len, cur_len;
+     int iov_offset = 0;
       struct cifs_sb_info *cifs_sb;
       struct cifsTconInfo *pTcon;
-     int xid;
       struct cifsFileInfo *open_file;
-     char *smb_read_data;
-     char __user *current_offset;
       struct smb_com_read_rsp *pSMBr;
+     char *read_data;
+
+     if (!nr_segs)
+             return 0;
+
+     len = iov_length(iov, nr_segs);
+     if (!len)
+             return 0;
 
       xid = GetXid();
       cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
@@ -1700,19 +1707,18 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
               FreeXid(xid);
               return rc;
       }
+
       open_file = file->private_data;
       pTcon = tlink_tcon(open_file->tlink);
 
       if ((file->f_flags & O_ACCMODE) == O_WRONLY)
               cFYI(1, "attempting read on write only file instance");
 
-     for (total_read = 0, current_offset = read_data;
-          read_size > total_read;
-          total_read += bytes_read, current_offset += bytes_read) {
-             current_read_size = min_t(const int, read_size - total_read,
-                                       cifs_sb->rsize);
+     for (total_read = 0; total_read < len; total_read += bytes_read) {
+             cur_len = min_t(const size_t, len - total_read, cifs_sb->rsize);
               rc = -EAGAIN;
-             smb_read_data = NULL;
+             read_data = NULL;
+
               while (rc == -EAGAIN) {
                       int buf_type = CIFS_NO_BUFFER;
                       if (open_file->invalidHandle) {
@@ -1720,27 +1726,25 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                               if (rc != 0)
                                       break;
                       }
-                     rc = CIFSSMBRead(xid, pTcon,
-                                      open_file->netfid,
-                                      current_read_size, *poffset,
-                                      &bytes_read, &smb_read_data,
-                                      &buf_type);
-                     pSMBr = (struct smb_com_read_rsp *)smb_read_data;
-                     if (smb_read_data) {
-                             if (copy_to_user(current_offset,
-                                             smb_read_data +
-                                             4 /* RFC1001 length field */ +
-                                             le16_to_cpu(pSMBr->DataOffset),

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1737

Quoted 10 lines
fs/cifs/file.c +9 -1
-                                             bytes_read))
+                     rc = CIFSSMBRead(xid, pTcon, open_file->netfid,
+                                      cur_len, *poffset, &bytes_read,
+                                      &read_data, &buf_type);
+                     pSMBr = (struct smb_com_read_rsp *)read_data;
+                     if (read_data) {
+                             char *data_offset = read_data + 4 +
+                                             le16_to_cpu(pSMBr->DataOffset);
+                             if (memcpy_toiovecend(iov, data_offset,
+                                                   iov_offset, bytes_read))

                                       ^^^^^
                       Much better. It's always better to use existing infrastructure.

Quoted 38 lines
fs/cifs/file.c +18 -4
                                       rc = -EFAULT;
-
                               if (buf_type == CIFS_SMALL_BUFFER)
-                                     cifs_small_buf_release(smb_read_data);
+                                     cifs_small_buf_release(read_data);
                               else if (buf_type == CIFS_LARGE_BUFFER)
-                                     cifs_buf_release(smb_read_data);
-                             smb_read_data = NULL;
+                                     cifs_buf_release(read_data);
+                             read_data = NULL;
+                             iov_offset += bytes_read;
                       }
               }
+
               if (rc || (bytes_read == 0)) {
                       if (total_read) {
                               break;
@@ -1753,13 +1757,57 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                       *poffset += bytes_read;
               }
       }
+
       FreeXid(xid);
       return total_read;
  }
 
+ssize_t cifs_user_read(struct file *file, char __user *read_data,
+                    size_t read_size, loff_t *poffset)
+{
+     struct iovec iov;
+     iov.iov_base = read_data;
+     iov.iov_len = read_size;
+
+     return cifs_iovec_read(file, &iov, 1, poffset);
+}
+
+ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                     unsigned long nr_segs, loff_t pos)

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1786

Quoted 10 lines
fs/cifs/file.c +10 -0
+{
+     ssize_t read;
+
+     read = cifs_iovec_read(iocb->ki_filp, iov, nr_segs, &pos);
+     if (read > 0)
+             iocb->ki_pos = pos;
+
+     return read;
+}
+

Again, I'd make the above static. If you do that, the compiler will likely just inline the function into the only caller below.

Quoted 18 lines
fs/cifs/file.c +18 -0
+ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                       unsigned long nr_segs, loff_t pos)
+{
+     struct inode *inode;
+
+     inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+     if (CIFS_I(inode)->clientCanCacheRead)
+             return generic_file_aio_read(iocb, iov, nr_segs, pos);
+
+     /*
+      * In strict cache mode we need to read from the server all the time
+      * if we don't have level II oplock because the server can delay mtime
+      * change - so we can't make a decision about inode invalidating.
+      * And we can also fail with pagereading if there are mandatory locks
+      * on pages affected by this read but not on the region from pos to
+      * pos+len-1.
+      */

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1813

Quoted 10 lines
fs/cifs/file.c +4 -1
+
+     return cifs_user_readv(iocb, iov, nr_segs, pos);
+}
 
  static ssize_t cifs_read(struct file *file, char *read_data, size_t read_size,
-     loff_t *poffset)
+                      loff_t *poffset)
  {
       int rc = -EACCES;
       unsigned int bytes_read = 0;

Nice work. Other than making that function static, this looks good to me, so you can add my Reviewed-by once that's been fixed...

Quoted 3 lines

--
Jeff Layton <[email protected]>

Jeff, I want to add cifs_user_readv/cifs_user_writev to direct_ops structure further - that's why I did these functions global.

Signature · 2 lines
Best regards,
Pavel Shilovsky.

A message that is not in the archive (sent off-list or not yet received): AANLkTimnraAMo=t0sPT5+wfKw3wp6YE2J0zXuQKUwagO-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#15 lore raw
Quoted 237 lines · On Mon, 27 Dec 2010 18:38:56 +0300 Pavel Shilovsky…

2010/12/27 Jeff Layton <[email protected]>:

Quoted 236 lines
Quoted 38 lines · On Sun, 26 Dec 2010 13:04:17 +0300 Pavel Shilovsky…

Read from the cache if we have at least Level II oplock - otherwise read from the server. Add cifs_user_readv to let the client read into iovec buffers.

---
 fs/cifs/cifsfs.c |    4 +-
 fs/cifs/cifsfs.h |    6 ++-
 fs/cifs/file.c   |  112 ++++++++++++++++++++++++++++++++++++++---------------

 3 files changed, 87 insertions(+), 35 deletions(-)
fs/cifs/cifsfs.c +2 -2
index 2b52020..0d6b005 100644
@@ -714,7 +714,7 @@ const struct file_operations cifs_file_ops = {
  const struct file_operations cifs_file_strict_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
@@ -770,7 +770,7 @@ const struct file_operations cifs_file_nobrl_ops = {
  const struct file_operations cifs_file_strict_nobrl_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
fs/cifs/cifsfs.h +0 -0
index 657738f..6ff3ca2 100644

fs/cifs/cifsfs.h extern int cifs_open(struct inode *inode, struct file *file); · line 86

Quoted 7 lines
fs/cifs/cifsfs.h +3 -1
@@ -81,7 +81,11 @@ extern int cifs_open(struct inode *inode, struct file *file);
  extern int cifs_close(struct inode *inode, struct file *file);
  extern int cifs_closedir(struct inode *inode, struct file *file);
  extern ssize_t cifs_user_read(struct file *file, char __user *read_data,
-                      size_t read_size, loff_t *poffset);
+                           size_t read_size, loff_t *poffset);
+extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                            unsigned long nr_segs, loff_t pos);

               ^^^^
       this can be static. There are no callers of cifs_user_readv.

Quoted 86 lines
fs/cifs/cifsfs.h +2 -0
+extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                              unsigned long nr_segs, loff_t pos);
  extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
                        size_t write_size, loff_t *poffset);
  extern int cifs_lock(struct file *, int, struct file_lock *);
fs/cifs/file.c +21 -26
index d48723c..c3bc65c 100644
@@ -1677,20 +1677,27 @@ int cifs_flush(struct file *file, fl_owner_t id)
       return rc;
  }
 
-ssize_t cifs_user_read(struct file *file, char __user *read_data,
-     size_t read_size, loff_t *poffset)
+static ssize_t
+cifs_iovec_read(struct file *file, const struct iovec *iov,
+              unsigned long nr_segs, loff_t *poffset)
  {
-     int rc = -EACCES;
-     unsigned int bytes_read = 0;
-     unsigned int total_read = 0;
-     unsigned int current_read_size;
+     int rc;
+     int xid;
+     unsigned int total_read, bytes_read = 0;
+     size_t len, cur_len;
+     int iov_offset = 0;
       struct cifs_sb_info *cifs_sb;
       struct cifsTconInfo *pTcon;
-     int xid;
       struct cifsFileInfo *open_file;
-     char *smb_read_data;
-     char __user *current_offset;
       struct smb_com_read_rsp *pSMBr;
+     char *read_data;
+
+     if (!nr_segs)
+             return 0;
+
+     len = iov_length(iov, nr_segs);
+     if (!len)
+             return 0;
 
       xid = GetXid();
       cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
@@ -1700,19 +1707,18 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
               FreeXid(xid);
               return rc;
       }
+
       open_file = file->private_data;
       pTcon = tlink_tcon(open_file->tlink);
 
       if ((file->f_flags & O_ACCMODE) == O_WRONLY)
               cFYI(1, "attempting read on write only file instance");
 
-     for (total_read = 0, current_offset = read_data;
-          read_size > total_read;
-          total_read += bytes_read, current_offset += bytes_read) {
-             current_read_size = min_t(const int, read_size - total_read,
-                                       cifs_sb->rsize);
+     for (total_read = 0; total_read < len; total_read += bytes_read) {
+             cur_len = min_t(const size_t, len - total_read, cifs_sb->rsize);
               rc = -EAGAIN;
-             smb_read_data = NULL;
+             read_data = NULL;
+
               while (rc == -EAGAIN) {
                       int buf_type = CIFS_NO_BUFFER;
                       if (open_file->invalidHandle) {
@@ -1720,27 +1726,25 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                               if (rc != 0)
                                       break;
                       }
-                     rc = CIFSSMBRead(xid, pTcon,
-                                      open_file->netfid,
-                                      current_read_size, *poffset,
-                                      &bytes_read, &smb_read_data,
-                                      &buf_type);
-                     pSMBr = (struct smb_com_read_rsp *)smb_read_data;
-                     if (smb_read_data) {
-                             if (copy_to_user(current_offset,
-                                             smb_read_data +
-                                             4 /* RFC1001 length field */ +
-                                             le16_to_cpu(pSMBr->DataOffset),

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1737

Quoted 10 lines
fs/cifs/file.c +9 -1
-                                             bytes_read))
+                     rc = CIFSSMBRead(xid, pTcon, open_file->netfid,
+                                      cur_len, *poffset, &bytes_read,
+                                      &read_data, &buf_type);
+                     pSMBr = (struct smb_com_read_rsp *)read_data;
+                     if (read_data) {
+                             char *data_offset = read_data + 4 +
+                                             le16_to_cpu(pSMBr->DataOffset);
+                             if (memcpy_toiovecend(iov, data_offset,
+                                                   iov_offset, bytes_read))

                                       ^^^^^
                       Much better. It's always better to use existing infrastructure.

Quoted 38 lines
fs/cifs/file.c +18 -4
                                       rc = -EFAULT;
-
                               if (buf_type == CIFS_SMALL_BUFFER)
-                                     cifs_small_buf_release(smb_read_data);
+                                     cifs_small_buf_release(read_data);
                               else if (buf_type == CIFS_LARGE_BUFFER)
-                                     cifs_buf_release(smb_read_data);
-                             smb_read_data = NULL;
+                                     cifs_buf_release(read_data);
+                             read_data = NULL;
+                             iov_offset += bytes_read;
                       }
               }
+
               if (rc || (bytes_read == 0)) {
                       if (total_read) {
                               break;
@@ -1753,13 +1757,57 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                       *poffset += bytes_read;
               }
       }
+
       FreeXid(xid);
       return total_read;
  }
 
+ssize_t cifs_user_read(struct file *file, char __user *read_data,
+                    size_t read_size, loff_t *poffset)
+{
+     struct iovec iov;
+     iov.iov_base = read_data;
+     iov.iov_len = read_size;
+
+     return cifs_iovec_read(file, &iov, 1, poffset);
+}
+
+ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                     unsigned long nr_segs, loff_t pos)

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1786

Quoted 10 lines
fs/cifs/file.c +10 -0
+{
+     ssize_t read;
+
+     read = cifs_iovec_read(iocb->ki_filp, iov, nr_segs, &pos);
+     if (read > 0)
+             iocb->ki_pos = pos;
+
+     return read;
+}
+

Again, I'd make the above static. If you do that, the compiler will likely just inline the function into the only caller below.

Quoted 18 lines
fs/cifs/file.c +18 -0
+ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                       unsigned long nr_segs, loff_t pos)
+{
+     struct inode *inode;
+
+     inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+     if (CIFS_I(inode)->clientCanCacheRead)
+             return generic_file_aio_read(iocb, iov, nr_segs, pos);
+
+     /*
+      * In strict cache mode we need to read from the server all the time
+      * if we don't have level II oplock because the server can delay mtime
+      * change - so we can't make a decision about inode invalidating.
+      * And we can also fail with pagereading if there are mandatory locks
+      * on pages affected by this read but not on the region from pos to
+      * pos+len-1.
+      */

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1813

Quoted 10 lines
fs/cifs/file.c +4 -1
+
+     return cifs_user_readv(iocb, iov, nr_segs, pos);
+}
 
  static ssize_t cifs_read(struct file *file, char *read_data, size_t read_size,
-     loff_t *poffset)
+                      loff_t *poffset)
  {
       int rc = -EACCES;
       unsigned int bytes_read = 0;

Nice work. Other than making that function static, this looks good to me, so you can add my Reviewed-by once that's been fixed...

Quoted 7 lines
Quoted 3 lines

--
Jeff Layton <[email protected]>

Jeff, I want to add cifs_user_readv/cifs_user_writev to direct_ops structure further - that's why I did these functions global.

Makes sense, but in general, it would be better to go ahead and make it static and then convert it to a global when you have callers that need it. That way we're not left with extra global symbols if that conversion gets put off for some reason. Linux CIFS is rife with examples of future plans that didn't pan out. :)

Signature · 1 line
Jeff Layton <[email protected]>

A message that is not in the archive (sent off-list or not yet received): 20101227110328.1d3032a1-9yPaYZwiELC+kQycOl6kW4xkIHaj4LzF@public.gmane.org

#17 lore raw

2010/12/27 Jeff Layton <[email protected]>:

Quoted 248 lines
Quoted 237 lines · On Mon, 27 Dec 2010 18:38:56 +0300 Pavel Shilovsky…

2010/12/27 Jeff Layton <[email protected]>:

Quoted 236 lines
Quoted 38 lines · On Sun, 26 Dec 2010 13:04:17 +0300 Pavel Shilovsky…

Read from the cache if we have at least Level II oplock - otherwise read from the server. Add cifs_user_readv to let the client read into iovec buffers.

---
 fs/cifs/cifsfs.c |    4 +-
 fs/cifs/cifsfs.h |    6 ++-
 fs/cifs/file.c   |  112 ++++++++++++++++++++++++++++++++++++++---------------

 3 files changed, 87 insertions(+), 35 deletions(-)
fs/cifs/cifsfs.c +2 -2
index 2b52020..0d6b005 100644
@@ -714,7 +714,7 @@ const struct file_operations cifs_file_ops = {
  const struct file_operations cifs_file_strict_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
@@ -770,7 +770,7 @@ const struct file_operations cifs_file_nobrl_ops = {
  const struct file_operations cifs_file_strict_nobrl_ops = {
       .read = do_sync_read,
       .write = do_sync_write,
-     .aio_read = generic_file_aio_read,
+     .aio_read = cifs_strict_readv,
       .aio_write = cifs_file_aio_write,
       .open = cifs_open,
       .release = cifs_close,
fs/cifs/cifsfs.h +0 -0
index 657738f..6ff3ca2 100644

fs/cifs/cifsfs.h extern int cifs_open(struct inode *inode, struct file *file); · line 86

Quoted 7 lines
fs/cifs/cifsfs.h +3 -1
@@ -81,7 +81,11 @@ extern int cifs_open(struct inode *inode, struct file *file);
  extern int cifs_close(struct inode *inode, struct file *file);
  extern int cifs_closedir(struct inode *inode, struct file *file);
  extern ssize_t cifs_user_read(struct file *file, char __user *read_data,
-                      size_t read_size, loff_t *poffset);
+                           size_t read_size, loff_t *poffset);
+extern ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                            unsigned long nr_segs, loff_t pos);

               ^^^^
       this can be static. There are no callers of cifs_user_readv.

Quoted 86 lines
fs/cifs/cifsfs.h +2 -0
+extern ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                              unsigned long nr_segs, loff_t pos);
  extern ssize_t cifs_user_write(struct file *file, const char __user *write_data,
                        size_t write_size, loff_t *poffset);
  extern int cifs_lock(struct file *, int, struct file_lock *);
fs/cifs/file.c +21 -26
index d48723c..c3bc65c 100644
@@ -1677,20 +1677,27 @@ int cifs_flush(struct file *file, fl_owner_t id)
       return rc;
  }
 
-ssize_t cifs_user_read(struct file *file, char __user *read_data,
-     size_t read_size, loff_t *poffset)
+static ssize_t
+cifs_iovec_read(struct file *file, const struct iovec *iov,
+              unsigned long nr_segs, loff_t *poffset)
  {
-     int rc = -EACCES;
-     unsigned int bytes_read = 0;
-     unsigned int total_read = 0;
-     unsigned int current_read_size;
+     int rc;
+     int xid;
+     unsigned int total_read, bytes_read = 0;
+     size_t len, cur_len;
+     int iov_offset = 0;
       struct cifs_sb_info *cifs_sb;
       struct cifsTconInfo *pTcon;
-     int xid;
       struct cifsFileInfo *open_file;
-     char *smb_read_data;
-     char __user *current_offset;
       struct smb_com_read_rsp *pSMBr;
+     char *read_data;
+
+     if (!nr_segs)
+             return 0;
+
+     len = iov_length(iov, nr_segs);
+     if (!len)
+             return 0;
 
       xid = GetXid();
       cifs_sb = CIFS_SB(file->f_path.dentry->d_sb);
@@ -1700,19 +1707,18 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
               FreeXid(xid);
               return rc;
       }
+
       open_file = file->private_data;
       pTcon = tlink_tcon(open_file->tlink);
 
       if ((file->f_flags & O_ACCMODE) == O_WRONLY)
               cFYI(1, "attempting read on write only file instance");
 
-     for (total_read = 0, current_offset = read_data;
-          read_size > total_read;
-          total_read += bytes_read, current_offset += bytes_read) {
-             current_read_size = min_t(const int, read_size - total_read,
-                                       cifs_sb->rsize);
+     for (total_read = 0; total_read < len; total_read += bytes_read) {
+             cur_len = min_t(const size_t, len - total_read, cifs_sb->rsize);
               rc = -EAGAIN;
-             smb_read_data = NULL;
+             read_data = NULL;
+
               while (rc == -EAGAIN) {
                       int buf_type = CIFS_NO_BUFFER;
                       if (open_file->invalidHandle) {
@@ -1720,27 +1726,25 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                               if (rc != 0)
                                       break;
                       }
-                     rc = CIFSSMBRead(xid, pTcon,
-                                      open_file->netfid,
-                                      current_read_size, *poffset,
-                                      &bytes_read, &smb_read_data,
-                                      &buf_type);
-                     pSMBr = (struct smb_com_read_rsp *)smb_read_data;
-                     if (smb_read_data) {
-                             if (copy_to_user(current_offset,
-                                             smb_read_data +
-                                             4 /* RFC1001 length field */ +
-                                             le16_to_cpu(pSMBr->DataOffset),

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1737

Quoted 10 lines
fs/cifs/file.c +9 -1
-                                             bytes_read))
+                     rc = CIFSSMBRead(xid, pTcon, open_file->netfid,
+                                      cur_len, *poffset, &bytes_read,
+                                      &read_data, &buf_type);
+                     pSMBr = (struct smb_com_read_rsp *)read_data;
+                     if (read_data) {
+                             char *data_offset = read_data + 4 +
+                                             le16_to_cpu(pSMBr->DataOffset);
+                             if (memcpy_toiovecend(iov, data_offset,
+                                                   iov_offset, bytes_read))

                                       ^^^^^
                       Much better. It's always better to use existing infrastructure.

Quoted 38 lines
fs/cifs/file.c +18 -4
                                       rc = -EFAULT;
-
                               if (buf_type == CIFS_SMALL_BUFFER)
-                                     cifs_small_buf_release(smb_read_data);
+                                     cifs_small_buf_release(read_data);
                               else if (buf_type == CIFS_LARGE_BUFFER)
-                                     cifs_buf_release(smb_read_data);
-                             smb_read_data = NULL;
+                                     cifs_buf_release(read_data);
+                             read_data = NULL;
+                             iov_offset += bytes_read;
                       }
               }
+
               if (rc || (bytes_read == 0)) {
                       if (total_read) {
                               break;
@@ -1753,13 +1757,57 @@ ssize_t cifs_user_read(struct file *file, char __user *read_data,
                       *poffset += bytes_read;
               }
       }
+
       FreeXid(xid);
       return total_read;
  }
 
+ssize_t cifs_user_read(struct file *file, char __user *read_data,
+                    size_t read_size, loff_t *poffset)
+{
+     struct iovec iov;
+     iov.iov_base = read_data;
+     iov.iov_len = read_size;
+
+     return cifs_iovec_read(file, &iov, 1, poffset);
+}
+
+ssize_t cifs_user_readv(struct kiocb *iocb, const struct iovec *iov,
+                     unsigned long nr_segs, loff_t pos)

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1786

Quoted 10 lines
fs/cifs/file.c +10 -0
+{
+     ssize_t read;
+
+     read = cifs_iovec_read(iocb->ki_filp, iov, nr_segs, &pos);
+     if (read > 0)
+             iocb->ki_pos = pos;
+
+     return read;
+}
+

Again, I'd make the above static. If you do that, the compiler will likely just inline the function into the only caller below.

Quoted 18 lines
fs/cifs/file.c +18 -0
+ssize_t cifs_strict_readv(struct kiocb *iocb, const struct iovec *iov,
+                       unsigned long nr_segs, loff_t pos)
+{
+     struct inode *inode;
+
+     inode = iocb->ki_filp->f_path.dentry->d_inode;
+
+     if (CIFS_I(inode)->clientCanCacheRead)
+             return generic_file_aio_read(iocb, iov, nr_segs, pos);
+
+     /*
+      * In strict cache mode we need to read from the server all the time
+      * if we don't have level II oplock because the server can delay mtime
+      * change - so we can't make a decision about inode invalidating.
+      * And we can also fail with pagereading if there are mandatory locks
+      * on pages affected by this read but not on the region from pos to
+      * pos+len-1.
+      */

fs/cifs/file.c ssize_t cifs_user_read(struct file *file, char __user *read_data, · line 1813

Quoted 10 lines
fs/cifs/file.c +4 -1
+
+     return cifs_user_readv(iocb, iov, nr_segs, pos);
+}
 
  static ssize_t cifs_read(struct file *file, char *read_data, size_t read_size,
-     loff_t *poffset)
+                      loff_t *poffset)
  {
       int rc = -EACCES;
       unsigned int bytes_read = 0;

Nice work. Other than making that function static, this looks good to me, so you can add my Reviewed-by once that's been fixed...

Quoted 7 lines
Quoted 3 lines

--
Jeff Layton <[email protected]>

Jeff, I want to add cifs_user_readv/cifs_user_writev to direct_ops structure further - that's why I did these functions global.

Quoted 9 lines

Makes sense, but in general, it would be better to go ahead and make it static and then convert it to a global when you have callers that need it. That way we're not left with extra global symbols if that conversion gets put off for some reason. Linux CIFS is rife with examples of future plans that didn't pan out. :)

--
Jeff Layton <[email protected]>

Ok, no problem. I will fix this.

Signature · 2 lines
Best regards,
Pavel Shilovsky.