Skip to contentSkip to conversation

Recvfile patch used for Samba.

PATCH Jeremy Allison 5 participants 8 messages Jul 22, 2013 – Jul 26, 2013

Subsystems (9)
lore
Share & tools
lore
#0 lore raw

Hi Steve and Jeff (and others).

Here is a patch that Samba vendors have been using to implement recvfile (copy directly from socket to file). It can improve write performance on boxes by a significant amount (10% or more).

I'm not qualified to evaluate this code, can someone who is (hi there Steve and Jeff :-) take a look at this and see if it's work shepherding into the kernel ?

Cheers,

Jeremy.

fs/splice.c +158 -0
diff -urp linux-2.6.37-rc5.orig/fs/splice.c linux-2.6.37-rc5/fs/splice.c
@@ -31,6 +31,7 @@
 #include <linux/uio.h>
 #include <linux/security.h>
 #include <linux/gfp.h>
+#include <net/sock.h>
 
 /*
  * Attempt to steal a page from a pipe buffer. This should perhaps go into
@@ -1387,6 +1388,141 @@ static long do_splice(struct file *in, l
 	return -EINVAL;
 }
 
+static ssize_t do_splice_from_socket(struct file *file, struct socket *sock,
+				     loff_t __user *ppos, size_t count)
+{		
+	struct address_space *mapping = file->f_mapping;
+	struct inode	*inode = mapping->host;
+	loff_t pos;
+	int count_tmp;
+	int err = 0;
+	int cPagePtr = 0;		
+	int cPagesAllocated = 0;
+	struct recvfile_ctl_blk rv_cb[MAX_PAGES_PER_RECVFILE];
+	struct kvec iov[MAX_PAGES_PER_RECVFILE];
+	struct msghdr msg;
+	long rcvtimeo;
+	int ret;
+
+	if(copy_from_user(&pos, ppos, sizeof(loff_t)))
+		return -EFAULT;
+
+	if(count > MAX_PAGES_PER_RECVFILE * PAGE_SIZE) {
+		printk("%s: count(%u) exceeds maxinum\n", __func__, count);
+		return -EINVAL;
+	}    
+	mutex_lock(&inode->i_mutex);
+
+	vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
+
+	/* We can write back this queue in page reclaim */
+	current->backing_dev_info = mapping->backing_dev_info;
+
+	err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
+	if (err != 0 || count == 0)
+		goto done;
+
+	file_remove_suid(file);
+	file_update_time(file);	
+
+	count_tmp = count;
+	do {
+		unsigned long bytes;	/* Bytes to write to page */
+		unsigned long offset;	/* Offset into pagecache page */
+		struct page *pageP;
+		void *fsdata;
+
+		offset = (pos & (PAGE_CACHE_SIZE - 1));
+		bytes = PAGE_CACHE_SIZE - offset;
+		if (bytes > count_tmp)
+			bytes = count_tmp;
+		ret = mapping->a_ops->write_begin(file, mapping, pos, bytes,
+						  AOP_FLAG_UNINTERRUPTIBLE,
+						  &pageP, &fsdata);
+
+		if (unlikely(ret)) {
+			err = ret;
+			for(cPagePtr = 0; cPagePtr < cPagesAllocated; cPagePtr++) {
+				kunmap(rv_cb[cPagePtr].rv_page);
+				ret = mapping->a_ops->write_end(file, mapping,
+								rv_cb[cPagePtr].rv_pos,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_page,
+								rv_cb[cPagePtr].rv_fsdata);
+			}
+			goto done;
+		}
+		rv_cb[cPagesAllocated].rv_page = pageP;
+		rv_cb[cPagesAllocated].rv_pos = pos;
+		rv_cb[cPagesAllocated].rv_count = bytes;
+		rv_cb[cPagesAllocated].rv_fsdata = fsdata;
+		iov[cPagesAllocated].iov_base = kmap(pageP) + offset;
+		iov[cPagesAllocated].iov_len = bytes;
+		cPagesAllocated++;
+		count_tmp -= bytes;
+		pos += bytes;
+	} while (count_tmp);
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;
+
+	ret = kernel_recvmsg(sock, &msg, &iov[0], cPagesAllocated, count,
+			     MSG_WAITALL | MSG_NOCATCHSIG);
+
+	sock->sk->sk_rcvtimeo = rcvtimeo;
+	if(ret != count)
+		err = -EPIPE;
+	else
+		err = 0;
+
+	if (unlikely(err < 0)) {
+		for(cPagePtr = 0; cPagePtr < cPagesAllocated; cPagePtr++) {
+			kunmap(rv_cb[cPagePtr].rv_page);
+			ret = mapping->a_ops->write_end(file, mapping,
+							rv_cb[cPagePtr].rv_pos,
+							rv_cb[cPagePtr].rv_count,
+							rv_cb[cPagePtr].rv_count,
+							rv_cb[cPagePtr].rv_page,
+							rv_cb[cPagePtr].rv_fsdata);
+		}
+		goto done;
+	}
+
+	for(cPagePtr=0,count=0;cPagePtr < cPagesAllocated;cPagePtr++) {
+		//flush_dcache_page(pageP);
+		kunmap(rv_cb[cPagePtr].rv_page);
+		ret = mapping->a_ops->write_end(file, mapping,
+						rv_cb[cPagePtr].rv_pos,
+						rv_cb[cPagePtr].rv_count,
+						rv_cb[cPagePtr].rv_count,
+						rv_cb[cPagePtr].rv_page,
+						rv_cb[cPagePtr].rv_fsdata);
+		if (unlikely(ret < 0))
+			printk("%s: write_end fail,ret = %d\n", __func__, ret);
+		count += rv_cb[cPagePtr].rv_count;
+		//cond_resched();
+	}
+	balance_dirty_pages_ratelimited_nr(mapping, cPagesAllocated);
+	copy_to_user(ppos,&pos,sizeof(loff_t));
+    
+done:
+	current->backing_dev_info = NULL;    
+	mutex_unlock(&inode->i_mutex);
+	if(err)
+		return err;
+	else 
+		return count;
+}
+
 /*
  * Map an iov into an array of pages and offset/length tupples. With the
  * partial_page structure, we can map several non-contiguous ranges into
@@ -1698,11 +1834,33 @@ SYSCALL_DEFINE6(splice, int, fd_in, loff
 	long error;
 	struct file *in, *out;
 	int fput_in, fput_out;
+	struct socket *sock = NULL;
 
 	if (unlikely(!len))
 		return 0;
 
 	error = -EBADF;
+
+	/* check if fd_in is a socket */
+	sock = sockfd_lookup(fd_in, &error);
+	if (sock) {
+		out = NULL;
+		if (!sock->sk)
+			goto done;
+		out = fget_light(fd_out, &fput_out);
+        
+		if (out) {
+			if (!(out->f_mode & FMODE_WRITE))
+				goto done;
+			error = do_splice_from_socket(out, sock, off_out, len);
+		}       
+done:
+		if(out)
+			fput_light(out, fput_out);      
+		fput(sock->file);
+		return error;
+	}
+
 	in = fget_light(fd_in, &fput_in);
 	if (in) {
 		if (in->f_mode & FMODE_READ) {
include/linux/fs.h +2 -0
diff -urp linux-2.6.37-rc5.orig/include/linux/fs.h linux-2.6.37-rc5/include/linux/fs.h
@@ -372,6 +372,8 @@ struct inodes_stat_t {
 #define SYNC_FILE_RANGE_WRITE		2
 #define SYNC_FILE_RANGE_WAIT_AFTER	4
 
+#define MAX_PAGES_PER_RECVFILE		32
+
 #ifdef __KERNEL__
 
 #include <linux/linkage.h>
include/linux/skbuff.h +3 -0
diff -urp linux-2.6.37-rc5.orig/include/linux/skbuff.h linux-2.6.37-rc5/include/linux/skbuff.h
@@ -1817,6 +1817,9 @@ extern unsigned int    datagram_poll(str
 extern int	       skb_copy_datagram_iovec(const struct sk_buff *from,
 					       int offset, struct iovec *to,
 					       int size);
+extern int	       skb_copy_datagram_to_kernel_iovec(const struct sk_buff *from,
+					       int offset, struct iovec *to,
+					       int size);
 extern int	       skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
 							int hlen,
 							struct iovec *iov);
include/linux/socket.h +3 -0
diff -urp linux-2.6.37-rc5.orig/include/linux/socket.h linux-2.6.37-rc5/include/linux/socket.h
@@ -261,6 +261,8 @@ struct ucred {
 #define MSG_NOSIGNAL	0x4000	/* Do not generate SIGPIPE */
 #define MSG_MORE	0x8000	/* Sender will send more */
 #define MSG_WAITFORONE	0x10000	/* recvmmsg(): block until 1+ packets avail */
+#define MSG_KERNSPACE	0x20000
+#define MSG_NOCATCHSIG	0x40000
 
 #define MSG_EOF         MSG_FIN
 
@@ -326,6 +328,7 @@ extern int verify_iovec(struct msghdr *m
 extern int memcpy_toiovec(struct iovec *v, unsigned char *kdata, int len);
 extern int memcpy_toiovecend(const struct iovec *v, unsigned char *kdata,
 			     int offset, int len);
+extern void memcpy_tokerneliovec(struct iovec *iov, unsigned char *kdata, int len);
 extern int move_addr_to_kernel(void __user *uaddr, int ulen, struct sockaddr *kaddr);
 extern int put_cmsg(struct msghdr*, int level, int type, int len, void *data);
 
include/linux/splice.h +8 -0
diff -urp linux-2.6.37-rc5.orig/include/linux/splice.h linux-2.6.37-rc5/include/linux/splice.h
@@ -57,6 +57,14 @@ struct splice_pipe_desc {
 	void (*spd_release)(struct splice_pipe_desc *, unsigned int);
 };
 
+struct recvfile_ctl_blk
+{
+	struct page *rv_page;
+	loff_t rv_pos;
+	size_t rv_count;
+	void *rv_fsdata;
+};
+
 typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
 			   struct splice_desc *);
 typedef int (splice_direct_actor)(struct pipe_inode_info *,
net/core/datagram.c +59 -0

65 lines not on this page · raw message

net/core/iovec.c +24 -0
diff -urp linux-2.6.37-rc5.orig/net/core/iovec.c linux-2.6.37-rc5/net/core/iovec.c
@@ -124,6 +124,30 @@ int memcpy_toiovecend(const struct iovec
 }
 EXPORT_SYMBOL(memcpy_toiovecend);
 
+/* This was removed in 2.6. Re-add it for splice from socket to file. */
+/*
+ *	In kernel copy to iovec. Returns -EFAULT on error.
+ *
+ *	Note: this modifies the original iovec.
+ */
+ 
+void memcpy_tokerneliovec(struct iovec *iov, unsigned char *kdata, int len)
+{
+	while(len>0)
+	{
+		if(iov->iov_len)
+		{
+			int copy = min_t(unsigned int, iov->iov_len, len);
+			memcpy(iov->iov_base, kdata, copy);
+			len -= copy;
+			kdata += copy;
+			iov->iov_base += copy;
+			iov->iov_len -= copy;
+		}
+		iov++;
+	}
+}
+
 /*
  *	Copy iovec to kernel. Returns -EFAULT on error.
  *
net/ipv4/tcp.c +22 -3
diff -urp linux-2.6.37-rc5.orig/net/ipv4/tcp.c linux-2.6.37-rc5/net/ipv4/tcp.c
@@ -1460,8 +1460,23 @@ int tcp_recvmsg(struct kiocb *iocb, stru
 	do {
 		u32 offset;
 
+		if (flags & MSG_NOCATCHSIG) {
+			if (signal_pending(current)) {
+				if (sigismember(&current->pending.signal, SIGQUIT) || 
+				    sigismember(&current->pending.signal, SIGABRT) ||
+				    sigismember(&current->pending.signal, SIGKILL) ||
+				    sigismember(&current->pending.signal, SIGTERM) ||
+				    sigismember(&current->pending.signal, SIGSTOP)) {
+
+					if (copied)
+						break;
+					copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
+					break;
+				}
+			}
+		}
 		/* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
-		if (tp->urg_data && tp->urg_seq == *seq) {
+		else if (tp->urg_data && tp->urg_seq == *seq) {
 			if (copied)
 				break;
 			if (signal_pending(current)) {
@@ -1690,8 +1705,12 @@ do_prequeue:
 			} else
 #endif
 			{
-				err = skb_copy_datagram_iovec(skb, offset,
-						msg->msg_iov, used);
+				if(msg->msg_flags & MSG_KERNSPACE)
+					err = skb_copy_datagram_to_kernel_iovec(skb,
+							offset, msg->msg_iov, used);
+				else
+					err = skb_copy_datagram_iovec(skb, offset,
+							msg->msg_iov, used);
 				if (err) {
 					/* Exception. Bailout! */
 					if (!copied)
#1 lore raw
Quoted 6 lines · Jeremy Allison

Here is a patch that Samba vendors have been using to implement recvfile (copy directly from socket to file). It can improve write performance on boxes by a significant amount (10% or more).

I'm not qualified to evaluate this code,

Nor I really. This doesn't apply to current btw.

Single comment: (just on a brief look)

do_splice_from_socket with 64bit compilation appears to have a pretty large (~1500 byte) stack frame which doesn't seem very nice. maybe a malloc/free instead of stack for

	struct recvfile_ctl_blk rv_cb[MAX_PAGES_PER_RECVFILE];
	struct kvec iov[MAX_PAGES_PER_RECVFILE];
#2 lore raw
In reply to Jeremy Allison
Quoted 6 lines · Jeremy Allison

Hi Steve and Jeff (and others).

Here is a patch that Samba vendors have been using to implement recvfile (copy directly from socket to file). It can improve write performance on boxes by a significant amount (10% or more).

Is that all? It's just moving the memcpy() into the kernel?

Quoted 3 lines

I'm not qualified to evaluate this code, can someone who is (hi there Steve and Jeff :-) take a look at this and see if it's work shepherding into the kernel ?

It's pretty nasty.

Quoted 20 lines

Cheers,

Jeremy.

diff -urp linux-2.6.37-rc5.orig/fs/splice.c linux-2.6.37-rc5/fs/splice.c

fs/splice.c +1 -0
@@ -31,6 +31,7 @@
 #include <linux/uio.h>
 #include <linux/security.h>
 #include <linux/gfp.h>
+#include <net/sock.h>
 
 /*
  * Attempt to steal a page from a pipe buffer. This should perhaps go into
@@ -1387,6 +1388,141 @@ static long do_splice(struct file *in, l
 	return -EINVAL;
 }
 

fs/splice.c static long do_splice(struct file *in, l · line 1400

Quoted 10 lines
fs/splice.c +10 -0
+static ssize_t do_splice_from_socket(struct file *file, struct socket *sock,
+				     loff_t __user *ppos, size_t count)
+{		
+	struct address_space *mapping = file->f_mapping;
+	struct inode	*inode = mapping->host;
+	loff_t pos;
+	int count_tmp;
+	int err = 0;
+	int cPagePtr = 0;		
+	int cPagesAllocated = 0;

camel case.

fs/splice.c static long do_splice(struct file *in, l · line 1401

Quoted 1 line
fs/splice.c +1 -0
+	struct recvfile_ctl_blk rv_cb[MAX_PAGES_PER_RECVFILE];

MAX_PAGES_PER_RECVFILE = 32, and the structure is 32 bytes on 64 bit. So 1k of stack usage right there. Bad.

fs/splice.c static long do_splice(struct file *in, l · line 1402

Quoted 1 line
fs/splice.c +1 -0
+	struct kvec iov[MAX_PAGES_PER_RECVFILE];

And that's another 512 bytes of stack. Way too much.

fs/splice.c static long do_splice(struct file *in, l · line 1403

Quoted 1 line
fs/splice.c +1 -0
+	struct msghdr msg;

Another ~52 bytes of stack.

fs/splice.c static long do_splice(struct file *in, l · line 1414

Quoted 11 lines
fs/splice.c +11 -0
+	long rcvtimeo;
+	int ret;
+
+	if(copy_from_user(&pos, ppos, sizeof(loff_t)))
+		return -EFAULT;
+
+	if(count > MAX_PAGES_PER_RECVFILE * PAGE_SIZE) {
+		printk("%s: count(%u) exceeds maxinum\n", __func__, count);
+		return -EINVAL;
+	}    
+	mutex_lock(&inode->i_mutex);

Locking on write needs to be passed to filesystems, not done by generic code. i.e. this isn't sufficient for XFS, and may also be problematic for btrfs.

fs/splice.c static long do_splice(struct file *in, l · line 1416

Quoted 2 lines
fs/splice.c +2 -0
+
+	vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);

That doesn't exist anymore - it was removed over a year ago....

fs/splice.c static long do_splice(struct file *in, l · line 1426

Quoted 10 lines
fs/splice.c +10 -0
+
+	/* We can write back this queue in page reclaim */
+	current->backing_dev_info = mapping->backing_dev_info;
+
+	err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
+	if (err != 0 || count == 0)
+		goto done;
+
+	file_remove_suid(file);
+	file_update_time(file);	

error checking?

fs/splice.c static long do_splice(struct file *in, l · line 1432

Quoted 6 lines
fs/splice.c +6 -0
+
+	count_tmp = count;
+	do {
+		unsigned long bytes;	/* Bytes to write to page */
+		unsigned long offset;	/* Offset into pagecache page */
+		struct page *pageP;

camel case

Quoted 23 lines
fs/splice.c +23 -0
+		void *fsdata;
+
+		offset = (pos & (PAGE_CACHE_SIZE - 1));
+		bytes = PAGE_CACHE_SIZE - offset;
+		if (bytes > count_tmp)
+			bytes = count_tmp;
+		ret = mapping->a_ops->write_begin(file, mapping, pos, bytes,
+						  AOP_FLAG_UNINTERRUPTIBLE,
+						  &pageP, &fsdata);
+
+		if (unlikely(ret)) {
+			err = ret;
+			for(cPagePtr = 0; cPagePtr < cPagesAllocated; cPagePtr++) {
+				kunmap(rv_cb[cPagePtr].rv_page);
+				ret = mapping->a_ops->write_end(file, mapping,
+								rv_cb[cPagePtr].rv_pos,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_page,
+								rv_cb[cPagePtr].rv_fsdata);
+			}
+			goto done;
+		}

fs/splice.c static long do_splice(struct file *in, l · line 1465

Quoted 10 lines
fs/splice.c +10 -0
+		rv_cb[cPagesAllocated].rv_page = pageP;
+		rv_cb[cPagesAllocated].rv_pos = pos;
+		rv_cb[cPagesAllocated].rv_count = bytes;
+		rv_cb[cPagesAllocated].rv_fsdata = fsdata;
+		iov[cPagesAllocated].iov_base = kmap(pageP) + offset;
+		iov[cPagesAllocated].iov_len = bytes;
+		cPagesAllocated++;
+		count_tmp -= bytes;
+		pos += bytes;
+	} while (count_tmp);

So, we are nesting up to 32 page locks here. That's bad. And we are nesting kmap() calls for all the pages individually - is that even safe to do?

So, what happens when we've got 16 pages in, and the filesystem has allocated space for those 16 blocks, and we get ENOSPC on the 17th? Sure, you undo the state here, but what about the 16 blocks that the filesystem has allocated to this file? There's no notification to the filesystem that they need to be truncated away because the write failed....

fs/splice.c static long do_splice(struct file *in, l · line 1476

Quoted 11 lines
fs/splice.c +11 -0
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;

We can hold the inode and the pages locked for 8 seconds?

I'll stop there. This is fundamentally broken. It's an attempt to do a multi-page write operation without any of the supporting structures needed to handle the failure cases properly. The nested page locking has "deadlock" written all over it, and the lack of partial failure handling shouts "data corruption" and "stale data exposure" to me. The fact it can block for up to 8 seconds waiting for network shenanigans to be completed while holding lots of locks is going to cause all sorts of problems under memory pressure.

Not to mention it means that all memory allocations in the msgrcv path need to be done with GFP_NOFS, because GFP_KERNEL allocations are almost guaranteed to deadlock on the locked pages this path already holds....

Need I say more?

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#3 lore raw
Quoted 134 lines · On Tue, 23 Jul 2013 17:10:27 +1000 Dave Chinner…
Quoted 6 lines · Jeremy Allison

Hi Steve and Jeff (and others).

Here is a patch that Samba vendors have been using to implement recvfile (copy directly from socket to file). It can improve write performance on boxes by a significant amount (10% or more).

Is that all? It's just moving the memcpy() into the kernel?

Quoted 3 lines

I'm not qualified to evaluate this code, can someone who is (hi there Steve and Jeff :-) take a look at this and see if it's work shepherding into the kernel ?

It's pretty nasty.

Quoted 20 lines

Cheers,

Jeremy.

diff -urp linux-2.6.37-rc5.orig/fs/splice.c linux-2.6.37-rc5/fs/splice.c

fs/splice.c +1 -0
@@ -31,6 +31,7 @@
 #include <linux/uio.h>
 #include <linux/security.h>
 #include <linux/gfp.h>
+#include <net/sock.h>
 
 /*
  * Attempt to steal a page from a pipe buffer. This should perhaps go into
@@ -1387,6 +1388,141 @@ static long do_splice(struct file *in, l
 	return -EINVAL;
 }
 

fs/splice.c static long do_splice(struct file *in, l · line 1400

Quoted 10 lines
fs/splice.c +10 -0
+static ssize_t do_splice_from_socket(struct file *file, struct socket *sock,
+				     loff_t __user *ppos, size_t count)
+{		
+	struct address_space *mapping = file->f_mapping;
+	struct inode	*inode = mapping->host;
+	loff_t pos;
+	int count_tmp;
+	int err = 0;
+	int cPagePtr = 0;		
+	int cPagesAllocated = 0;

camel case.

fs/splice.c static long do_splice(struct file *in, l · line 1401

Quoted 1 line
fs/splice.c +1 -0
+	struct recvfile_ctl_blk rv_cb[MAX_PAGES_PER_RECVFILE];

MAX_PAGES_PER_RECVFILE = 32, and the structure is 32 bytes on 64 bit. So 1k of stack usage right there. Bad.

fs/splice.c static long do_splice(struct file *in, l · line 1402

Quoted 1 line
fs/splice.c +1 -0
+	struct kvec iov[MAX_PAGES_PER_RECVFILE];

And that's another 512 bytes of stack. Way too much.

fs/splice.c static long do_splice(struct file *in, l · line 1403

Quoted 1 line
fs/splice.c +1 -0
+	struct msghdr msg;

Another ~52 bytes of stack.

fs/splice.c static long do_splice(struct file *in, l · line 1414

Quoted 11 lines
fs/splice.c +11 -0
+	long rcvtimeo;
+	int ret;
+
+	if(copy_from_user(&pos, ppos, sizeof(loff_t)))
+		return -EFAULT;
+
+	if(count > MAX_PAGES_PER_RECVFILE * PAGE_SIZE) {
+		printk("%s: count(%u) exceeds maxinum\n", __func__, count);
+		return -EINVAL;
+	}    
+	mutex_lock(&inode->i_mutex);

Locking on write needs to be passed to filesystems, not done by generic code. i.e. this isn't sufficient for XFS, and may also be problematic for btrfs.

fs/splice.c static long do_splice(struct file *in, l · line 1416

Quoted 2 lines
fs/splice.c +2 -0
+
+	vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);

That doesn't exist anymore - it was removed over a year ago....

fs/splice.c static long do_splice(struct file *in, l · line 1426

Quoted 10 lines
fs/splice.c +10 -0
+
+	/* We can write back this queue in page reclaim */
+	current->backing_dev_info = mapping->backing_dev_info;
+
+	err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
+	if (err != 0 || count == 0)
+		goto done;
+
+	file_remove_suid(file);
+	file_update_time(file);	

error checking?

fs/splice.c static long do_splice(struct file *in, l · line 1432

Quoted 6 lines
fs/splice.c +6 -0
+
+	count_tmp = count;
+	do {
+		unsigned long bytes;	/* Bytes to write to page */
+		unsigned long offset;	/* Offset into pagecache page */
+		struct page *pageP;

camel case

Quoted 29 lines
fs/splice.c +29 -0
+		void *fsdata;
+
+		offset = (pos & (PAGE_CACHE_SIZE - 1));
+		bytes = PAGE_CACHE_SIZE - offset;
+		if (bytes > count_tmp)
+			bytes = count_tmp;
+		ret = mapping->a_ops->write_begin(file, mapping, pos, bytes,
+						  AOP_FLAG_UNINTERRUPTIBLE,
+						  &pageP, &fsdata);
+
+		if (unlikely(ret)) {
+			err = ret;
+			for(cPagePtr = 0; cPagePtr < cPagesAllocated; cPagePtr++) {
+				kunmap(rv_cb[cPagePtr].rv_page);
+				ret = mapping->a_ops->write_end(file, mapping,
+								rv_cb[cPagePtr].rv_pos,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_count,
+								rv_cb[cPagePtr].rv_page,
+								rv_cb[cPagePtr].rv_fsdata);
+			}
+			goto done;
+		}
+		rv_cb[cPagesAllocated].rv_page = pageP;
+		rv_cb[cPagesAllocated].rv_pos = pos;
+		rv_cb[cPagesAllocated].rv_count = bytes;
+		rv_cb[cPagesAllocated].rv_fsdata = fsdata;
+		iov[cPagesAllocated].iov_base = kmap(pageP) + offset;
+		iov[cPagesAllocated].iov_len = bytes;
Quoted 9 lines

fs/splice.c static long do_splice(struct file *in, l · line 1465

Quoted 4 lines
fs/splice.c +4 -0
+		cPagesAllocated++;
+		count_tmp -= bytes;
+		pos += bytes;
+	} while (count_tmp);

So, we are nesting up to 32 page locks here. That's bad. And we are nesting kmap() calls for all the pages individually - is that even safe to do?

You can do it, but if you run out of kmap space, it'll hang and you'll be screwed. Wouldn't take much either -- just a few of these calls running in parallel would be enough to deadlock all of them.

Quoted 30 lines

So, what happens when we've got 16 pages in, and the filesystem has allocated space for those 16 blocks, and we get ENOSPC on the 17th? Sure, you undo the state here, but what about the 16 blocks that the filesystem has allocated to this file? There's no notification to the filesystem that they need to be truncated away because the write failed....

fs/splice.c static long do_splice(struct file *in, l · line 1476

Quoted 11 lines
fs/splice.c +11 -0
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;

We can hold the inode and the pages locked for 8 seconds?

I'll stop there. This is fundamentally broken. It's an attempt to do a multi-page write operation without any of the supporting structures needed to handle the failure cases properly. The nested page locking has "deadlock" written all over it, and the lack of partial failure handling shouts "data corruption" and "stale data exposure" to me. The fact it can block for up to 8 seconds waiting for network shenanigans to be completed while holding lots of locks is going to cause all sorts of problems under memory pressure.

Quoted 5 lines

Not to mention it means that all memory allocations in the msgrcv path need to be done with GFP_NOFS, because GFP_KERNEL allocations are almost guaranteed to deadlock on the locked pages this path already holds....

Yep, pretty broken.

A better approach might be to hook this call up to the sk_write_space callback on the socket. Have it block until there is something to actually be read, and then do the loop over the pages one at a time at that point until all of the ready data is read (non-blocking) into the pages. When you run out of data, unlock everything and go back to blocking until there is more or until you hit the "len" passed into the syscall.

Locking and error handling with that approach would still be pretty tricky, however...

Signature · 1 line
Jeff Layton <[email protected]>
#4 lore raw
In reply to Dave Chinner
Quoted 34 lines · Dave Chinner

So, we are nesting up to 32 page locks here. That's bad. And we are nesting kmap() calls for all the pages individually - is that even safe to do?

So, what happens when we've got 16 pages in, and the filesystem has allocated space for those 16 blocks, and we get ENOSPC on the 17th? Sure, you undo the state here, but what about the 16 blocks that the filesystem has allocated to this file? There's no notification to the filesystem that they need to be truncated away because the write failed....

fs/splice.c static long do_splice(struct file *in, l · line 1476

Quoted 11 lines
fs/splice.c +11 -0
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;

We can hold the inode and the pages locked for 8 seconds?

I'll stop there. This is fundamentally broken. It's an attempt to do a multi-page write operation without any of the supporting structures needed to handle the failure cases properly. The nested page locking has "deadlock" written all over it, and the lack of partial failure handling shouts "data corruption" and "stale data exposure" to me. The fact it can block for up to 8 seconds waiting for network shenanigans to be completed while holding lots of locks is going to cause all sorts of problems under memory pressure.

Quoted 6 lines

Not to mention it means that all memory allocations in the msgrcv path need to be done with GFP_NOFS, because GFP_KERNEL allocations are almost guaranteed to deadlock on the locked pages this path already holds....

Need I say more?

No, that's great ! :-).

Thanks for the analysis. I'd heard it wasn't near production quality, but not being a kernel engineer myself I wasn't able to make that assessment.

Having said that the OEMs that are using it does find it improves write speeds by a large amount (10% or more), so it's showing there is room for improvement here if the correct code can be created for recvfile.

Cheers,

Jeremy.

#5 lore raw
Quoted 44 lines · Jeremy Allison
Quoted 34 lines · Dave Chinner

So, we are nesting up to 32 page locks here. That's bad. And we are nesting kmap() calls for all the pages individually - is that even safe to do?

So, what happens when we've got 16 pages in, and the filesystem has allocated space for those 16 blocks, and we get ENOSPC on the 17th? Sure, you undo the state here, but what about the 16 blocks that the filesystem has allocated to this file? There's no notification to the filesystem that they need to be truncated away because the write failed....

Review

Quoted 11 lines
(quoted diff) +11 -0
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;

We can hold the inode and the pages locked for 8 seconds?

I'll stop there. This is fundamentally broken. It's an attempt to do a multi-page write operation without any of the supporting structures needed to handle the failure cases properly. The nested page locking has "deadlock" written all over it, and the lack of partial failure handling shouts "data corruption" and "stale data exposure" to me. The fact it can block for up to 8 seconds waiting for network shenanigans to be completed while holding lots of locks is going to cause all sorts of problems under memory pressure.

Quoted 6 lines

Not to mention it means that all memory allocations in the msgrcv path need to be done with GFP_NOFS, because GFP_KERNEL allocations are almost guaranteed to deadlock on the locked pages this path already holds....

Need I say more?

No, that's great ! :-).

Quoted 8 lines

Thanks for the analysis. I'd heard it wasn't near production quality, but not being a kernel engineer myself I wasn't able to make that assessment.

Having said that the OEMs that are using it does find it improves write speeds by a large amount (10% or more), so it's showing there is room for improvement here if the correct code can be created for recvfile.

10% is not very large gain given the complexity it adds, and I question that the gain actually comes from moving the memcpy() into the kernel. If this recvfile code enabled zero-copy behaviour into the page cache, then it would be worth pursuing. But it doesn't, and so IMO the complexity is not worth the gain right now.

Indeed, I suspect the 10% gain will be from the multi-page write behaviour that was hacked into the code. I wrote a multi-page write prototype ~3 years ago that showed write(2) performance gains of roughly 10% on low CPU power machines running XFS.

$ git branch |grep multi
  multipage-write
$ git checkout multipage-write 
Checking out files: 100% (45114/45114), done.
Switched to branch 'multipage-write'
$ head -4 Makefile 
VERSION = 2
PATCHLEVEL = 6
SUBLEVEL = 37
EXTRAVERSION = -rc6
$

I should probably pick this up again and push it forwards. FWIW, I've attached the first multipage-write infrastructure patch from the above branch to show how this sort of operation needs to be done from a filesystem and page-cache perspective to avoid locking problems have sane error handling.

I beleive the version that Christoph implemented for a couple of OEMs around that time de-multiplexed the ->iomap method....

Cheers,

Dave.

Signature · 11 lines
Dave Chinner
[email protected]

multipage-write: introduce iomap infrastructure

From: Dave Chinner <[email protected]>

Add infrastructure for multipage writes by defining the mapping interface
that the multipage writes will use and the main multipage write loop.

Signed-off-by: Dave Chinner <[email protected]>
include/linux/fs.h +4 -0
index 76041b6..1196877 100644
@@ -513,6 +513,7 @@ enum positive_aop_returns {
 struct page;
 struct address_space;
 struct writeback_control;
+struct iomap;
 
 struct iov_iter {
 	const struct iovec *iov;
@@ -604,6 +605,9 @@ struct address_space_operations {
 	int (*is_partially_uptodate) (struct page *, read_descriptor_t *,
 					unsigned long);
 	int (*error_remove_page)(struct address_space *, struct page *);
+
+	int (*iomap)(struct address_space *mapping, loff_t pos,
+			ssize_t length, struct iomap *iomap, int cmd);
 };
 
 /*
include/linux/iomap.h new file +45 -0

45 lines not on this page · raw message

mm/filemap.c +170 -9

226 lines not on this page · raw message

#6 lore raw

Hi,

Quoted 83 lines · Dave Chinner
Quoted 44 lines · Jeremy Allison
Quoted 34 lines · Dave Chinner

So, we are nesting up to 32 page locks here. That's bad. And we are nesting kmap() calls for all the pages individually - is that even safe to do?

So, what happens when we've got 16 pages in, and the filesystem has allocated space for those 16 blocks, and we get ENOSPC on the 17th? Sure, you undo the state here, but what about the 16 blocks that the filesystem has allocated to this file? There's no notification to the filesystem that they need to be truncated away because the write failed....

Review

Quoted 11 lines
(quoted diff) +11 -0
+
+	/* IOV is ready, receive the date from socket now */
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_iov = (struct iovec *)&iov[0];
+	msg.msg_iovlen = cPagesAllocated ;
+	msg.msg_control = NULL;
+	msg.msg_controllen = 0;
+	msg.msg_flags = MSG_KERNSPACE;
+	rcvtimeo = sock->sk->sk_rcvtimeo;    
+	sock->sk->sk_rcvtimeo = 8 * HZ;

We can hold the inode and the pages locked for 8 seconds?

I'll stop there. This is fundamentally broken. It's an attempt to do a multi-page write operation without any of the supporting structures needed to handle the failure cases properly. The nested page locking has "deadlock" written all over it, and the lack of partial failure handling shouts "data corruption" and "stale data exposure" to me. The fact it can block for up to 8 seconds waiting for network shenanigans to be completed while holding lots of locks is going to cause all sorts of problems under memory pressure.

Quoted 6 lines

Not to mention it means that all memory allocations in the msgrcv path need to be done with GFP_NOFS, because GFP_KERNEL allocations are almost guaranteed to deadlock on the locked pages this path already holds....

Need I say more?

No, that's great ! :-).

Quoted 8 lines

Thanks for the analysis. I'd heard it wasn't near production quality, but not being a kernel engineer myself I wasn't able to make that assessment.

Having said that the OEMs that are using it does find it improves write speeds by a large amount (10% or more), so it's showing there is room for improvement here if the correct code can be created for recvfile.

10% is not very large gain given the complexity it adds, and I question that the gain actually comes from moving the memcpy() into the kernel. If this recvfile code enabled zero-copy behaviour into the page cache, then it would be worth pursuing. But it doesn't, and so IMO the complexity is not worth the gain right now.

Indeed, I suspect the 10% gain will be from the multi-page write behaviour that was hacked into the code. I wrote a multi-page write prototype ~3 years ago that showed write(2) performance gains of roughly 10% on low CPU power machines running XFS.

$ git branch |grep multi
  multipage-write
$ git checkout multipage-write 
Checking out files: 100% (45114/45114), done.
Switched to branch 'multipage-write'
$ head -4 Makefile 
VERSION = 2
PATCHLEVEL = 6
SUBLEVEL = 37
EXTRAVERSION = -rc6
$

I should probably pick this up again and push it forwards. FWIW, I've attached the first multipage-write infrastructure patch from the above branch to show how this sort of operation needs to be done from a filesystem and page-cache perspective to avoid locking problems have sane error handling.

Quoted 6 lines

I beleive the version that Christoph implemented for a couple of OEMs around that time de-multiplexed the ->iomap method....

Cheers,

Dave.

I have Christoph's version here and between other tasks, I'm working on figuring out how it all works and writing GFS2 support for it. I'd more or less got that complete for your version, but there are a number of differences with Christoph's code and it is taking me a while to ensure that I've not missed any corner cases and figuring out how to fit some of GFS2's odd write modes into the framework,

Steve.

#7 lore raw
Quoted 19 lines · Steven Whitehouse

Hi,

Quoted 16 lines · Dave Chinner
Quoted 4 lines · Jeremy Allison

Having said that the OEMs that are using it does find it improves write speeds by a large amount (10% or more), so it's showing there is room for improvement here if the correct code can be created for recvfile.

10% is not very large gain given the complexity it adds, and I question that the gain actually comes from moving the memcpy() into the kernel. If this recvfile code enabled zero-copy behaviour into the page cache, then it would be worth pursuing. But it doesn't, and so IMO the complexity is not worth the gain right now.

Indeed, I suspect the 10% gain will be from the multi-page write behaviour that was hacked into the code. I wrote a multi-page write prototype ~3 years ago that showed write(2) performance gains of roughly 10% on low CPU power machines running XFS.

...

Quoted 9 lines
Quoted 8 lines

I should probably pick this up again and push it forwards. FWIW, I've attached the first multipage-write infrastructure patch from the above branch to show how this sort of operation needs to be done from a filesystem and page-cache perspective to avoid locking problems have sane error handling.

I beleive the version that Christoph implemented for a couple of OEMs around that time de-multiplexed the ->iomap method....

Quoted 6 lines

I have Christoph's version here and between other tasks, I'm working on figuring out how it all works and writing GFS2 support for it. I'd more or less got that complete for your version, but there are a number of differences with Christoph's code and it is taking me a while to ensure that I've not missed any corner cases and figuring out how to fit some of GFS2's odd write modes into the framework,

Can you send me Christoph's version so I can have a look at the differences? I'm pretty sure there isn't anything architecturally different, but I've never seen it so I don't know exactly how it differs...

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]