Skip to contentSkip to conversation

splice support #3

PATCH In review Jens Axboe 3 participants 6 messages Mar 30, 2006 – Mar 31, 2006

Subsystems (6)
lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

PATCH 1/1 splice support #3

Hi,

Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.

----

This adds support for the sys_splice system call. Using a pipe as a transport, it can connect to files or sockets (latter as output only).

arch/i386/kernel/syscall_table.S +1 -0
index 326595f..ce3ef4f 100644
@@ -312,3 +312,4 @@ ENTRY(sys_call_table)
 	.long sys_unshare		/* 310 */
 	.long sys_set_robust_list
 	.long sys_get_robust_list
+	.long sys_splice
arch/ia64/kernel/entry.S +1 -0
index 0e3eda9..750e8e7 100644
@@ -1605,5 +1605,6 @@ sys_call_table:
 	data8 sys_ni_syscall			// reserved for pselect
 	data8 sys_ni_syscall			// 1295 reserved for ppoll
 	data8 sys_unshare
+	data8 sys_splice
 
 	.org sys_call_table + 8*NR_syscalls	// guard against failures to increase NR_syscalls
fs/Makefile +1 -1
index 080b386..f3a4f70 100644
@@ -10,7 +10,7 @@ obj-y :=	open.o read_write.o file_table.
 		ioctl.o readdir.o select.o fifo.o locks.o dcache.o inode.o \
 		attr.o bad_inode.o file.o filesystems.o namespace.o aio.o \
 		seq_file.o xattr.o libfs.o fs-writeback.o mpage.o direct-io.o \
-		ioprio.o pnode.o drop_caches.o
+		ioprio.o pnode.o drop_caches.o splice.o
 
 obj-$(CONFIG_INOTIFY)		+= inotify.o
 obj-$(CONFIG_EPOLL)		+= eventpoll.o
fs/ext2/file.c +2 -0
index 509ccec..23e2c7c 100644
@@ -53,6 +53,8 @@ const struct file_operations ext2_file_o
 	.readv		= generic_file_readv,
 	.writev		= generic_file_writev,
 	.sendfile	= generic_file_sendfile,
+	.splice_read	= generic_file_splice_read,
+	.splice_write	= generic_file_splice_write,
 };
 
 #ifdef CONFIG_EXT2_FS_XIP
fs/ext3/file.c +2 -0
index 783a796..1efefb6 100644
@@ -119,6 +119,8 @@ const struct file_operations ext3_file_o
 	.release	= ext3_release_file,
 	.fsync		= ext3_sync_file,
 	.sendfile	= generic_file_sendfile,
+	.splice_read	= generic_file_splice_read,
+	.splice_write	= generic_file_splice_write,
 };
 
 struct inode_operations ext3_file_inode_operations = {
fs/pipe.c +28 -5
index e2f4f1d..2414bf2 100644
@@ -15,6 +15,7 @@ #include <linux/mount.h>
 #include <linux/pipe_fs_i.h>
 #include <linux/uio.h>
 #include <linux/highmem.h>
+#include <linux/pagemap.h>
 
 #include <asm/uaccess.h>
 #include <asm/ioctls.h>
@@ -94,11 +95,20 @@ static void anon_pipe_buf_release(struct
 {
 	struct page *page = buf->page;
 
-	if (info->tmp_page) {
-		__free_page(page);
+	/*
+	 * If nobody else uses this page, and we don't already have a
+	 * temporary page, let's keep track of it as a one-deep
+	 * allocation cache
+	 */
+	if (page_count(page) == 1 && !info->tmp_page) {
+		info->tmp_page = page;
 		return;
 	}
-	info->tmp_page = page;
+
+	/*
+	 * Otherwise just release our reference to it
+	 */
+	page_cache_release(page);
 }
 
 static void *anon_pipe_buf_map(struct file *file, struct pipe_inode_info *info, struct pipe_buffer *buf)
@@ -152,6 +162,11 @@ pipe_readv(struct file *filp, const stru
 				chars = total_len;
 
 			addr = ops->map(filp, info, buf);
+			if (IS_ERR(addr)) {
+				if (!ret)
+					ret = PTR_ERR(addr);
+				break;
+			}
 			error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars);
 			ops->unmap(info, buf);
 			if (unlikely(error)) {
@@ -254,8 +269,16 @@ pipe_writev(struct file *filp, const str
 		struct pipe_buf_operations *ops = buf->ops;
 		int offset = buf->offset + buf->len;
 		if (ops->can_merge && offset + chars <= PAGE_SIZE) {
-			void *addr = ops->map(filp, info, buf);
-			int error = pipe_iov_copy_from_user(offset + addr, iov, chars);
+			void *addr;
+			int error;
+
+			addr = ops->map(filp, info, buf);
+			if (IS_ERR(addr)) {
+				error = PTR_ERR(addr);
+				goto out;
+			}
+			error = pipe_iov_copy_from_user(offset + addr, iov,
+							chars);
 			ops->unmap(info, buf);
 			ret = error;
 			do_wakeup = 1;
fs/reiserfs/file.c +2 -0
index 010094d..cf6e1cf 100644
@@ -1576,6 +1576,8 @@ const struct file_operations reiserfs_fi
 	.sendfile = generic_file_sendfile,
 	.aio_read = generic_file_aio_read,
 	.aio_write = reiserfs_aio_write,
+	.splice_read = generic_file_splice_read,
+	.splice_write = generic_file_splice_write,
 };
 
 struct inode_operations reiserfs_file_inode_operations = {
fs/splice.c new file +612 -0

612 lines not on this page · raw message

include/asm-i386/unistd.h +2 -1
index 014e356..789e9bd 100644
@@ -318,8 +318,9 @@ #define __NR_ppoll		309
 #define __NR_unshare		310
 #define __NR_set_robust_list	311
 #define __NR_get_robust_list	312
+#define __NR_sys_splice		313
 
-#define NR_syscalls 313
+#define NR_syscalls 314
 
 /*
  * user-visible error numbers are in the range -1 - -128: see
include/asm-ia64/unistd.h +2 -1
index 019956c..36070c1 100644
@@ -285,12 +285,13 @@ #define __NR_fchmodat			1292
 #define __NR_faccessat			1293
 /* 1294, 1295 reserved for pselect/ppoll */
 #define __NR_unshare			1296
+#define __NR_splice			1297
 
 #ifdef __KERNEL__
 
 #include <linux/config.h>
 
-#define NR_syscalls			273 /* length of syscall table */
+#define NR_syscalls			274 /* length of syscall table */
 
 #define __ARCH_WANT_SYS_RT_SIGACTION
 
include/asm-powerpc/unistd.h +2 -1
index 1e99074..536ba08 100644
@@ -301,8 +301,9 @@ #define __NR_spu_create		279
 #define __NR_pselect6		280
 #define __NR_ppoll		281
 #define __NR_unshare		282
+#define __NR_splice		283
 
-#define __NR_syscalls		283
+#define __NR_syscalls		284
 
 #ifdef __KERNEL__
 #define __NR__exit __NR_exit
include/asm-x86_64/unistd.h +3 -1
index fcc5163..f21ff2c 100644
@@ -609,8 +609,10 @@ #define __NR_set_robust_list	273
 __SYSCALL(__NR_set_robust_list, sys_set_robust_list)
 #define __NR_get_robust_list	274
 __SYSCALL(__NR_get_robust_list, sys_get_robust_list)
+#define __NR_splice		275
+__SYSCALL(__NR_splice, sys_splice)
 
-#define __NR_syscall_max __NR_get_robust_list
+#define __NR_syscall_max __NR_splice
 
 #ifndef __NO_STUBS
 
include/linux/fs.h +4 -0
index 408fe89..20fa5f6 100644
@@ -1032,6 +1032,8 @@ struct file_operations {
 	int (*check_flags)(int);
 	int (*dir_notify)(struct file *filp, unsigned long arg);
 	int (*flock) (struct file *, int, struct file_lock *);
+	ssize_t (*splice_write)(struct inode *, struct file *, size_t, unsigned int);
+	ssize_t (*splice_read)(struct file *, struct inode *, size_t, unsigned int);
 };
 
 struct inode_operations {
@@ -1609,6 +1611,8 @@ extern ssize_t generic_file_sendfile(str
 extern void do_generic_mapping_read(struct address_space *mapping,
 				    struct file_ra_state *, struct file *,
 				    loff_t *, read_descriptor_t *, read_actor_t);
+extern ssize_t generic_file_splice_read(struct file *, struct inode *, size_t, unsigned int);
+extern ssize_t generic_file_splice_write(struct inode *, struct file *, size_t, unsigned int);
 extern void
 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
 extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 
include/linux/syscalls.h +2 -0
index e487e3b..e78ffc7 100644
@@ -569,5 +569,7 @@ asmlinkage long compat_sys_newfstatat(un
 asmlinkage long compat_sys_openat(unsigned int dfd, const char __user *filename,
 				   int flags, int mode);
 asmlinkage long sys_unshare(unsigned long unshare_flags);
+asmlinkage long sys_splice(int fdin, int fdout, size_t len,
+				unsigned int flags);
 
 #endif
net/socket.c +5 -1
index fcd77ea..b13042f 100644
@@ -119,7 +119,10 @@ static ssize_t sock_writev(struct file *
 static ssize_t sock_sendpage(struct file *file, struct page *page,
 			     int offset, size_t size, loff_t *ppos, int more);
 
+extern ssize_t generic_splice_sendpage(struct inode *inode, struct file *out,
+				size_t len, unsigned int flags);
 
+
 /*
  *	Socket files have a set of 'special' operations as well as the generic file ones. These don't appear
  *	in the operation structures but are done directly via the socketcall() multiplexor.
@@ -141,7 +144,8 @@ #endif
 	.fasync =	sock_fasync,
 	.readv =	sock_readv,
 	.writev =	sock_writev,
-	.sendpage =	sock_sendpage
+	.sendpage =	sock_sendpage,
+	.splice_write = generic_splice_sendpage,
 };
 
 /*
Signature · 1 line
Jens Axboe
author
#1 lore raw

PATCH 1/1 splice SPLICE_F_MOVE support

Hi,

This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the SPLICE_F_MOVE flag to the splice syscall to ask the kernel to try and move pages, if needed.

Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)

---

From: Jens Axboe <[email protected]>
Date: Thu Mar 30 15:16:46 2006 +0200
Subject: [PATCH] splice: add support for SPLICE_F_MOVE flag

This enables the caller to migrate pages from one address space page cache to another. In buzz word marketing, you can do zero-copy file copies!

---

 fs/pipe.c                 |    8 +++
 fs/splice.c               |  121 +++++++++++++++++++++++++++++++--------------
 include/linux/pipe_fs_i.h |    8 +++
 3 files changed, 100 insertions(+), 37 deletions(-)

54facc65189d86f4b8b7eb9be5c0c5623090483d

fs/pipe.c +8 -0
index 2414bf2..109a102 100644
@@ -121,11 +121,19 @@ static void anon_pipe_buf_unmap(struct p
 	kunmap(buf->page);
 }
 
+static int anon_pipe_buf_steal(struct pipe_inode_info *info,
+			       struct pipe_buffer *buf)
+{
+	buf->stolen = 1;
+	return 0;
+}
+
 static struct pipe_buf_operations anon_pipe_buf_ops = {
 	.can_merge = 1,
 	.map = anon_pipe_buf_map,
 	.unmap = anon_pipe_buf_unmap,
 	.release = anon_pipe_buf_release,
+	.steal = anon_pipe_buf_steal,
 };
 
 static ssize_t
fs/splice.c +84 -37

179 lines not on this page · raw message

include/linux/pipe_fs_i.h +8 -0
index b12e59c..75c7f55 100644
@@ -9,6 +9,7 @@ struct pipe_buffer {
 	struct page *page;
 	unsigned int offset, len;
 	struct pipe_buf_operations *ops;
+	unsigned int stolen;
 };
 
 struct pipe_buf_operations {
@@ -16,6 +17,7 @@ struct pipe_buf_operations {
 	void * (*map)(struct file *, struct pipe_inode_info *, struct pipe_buffer *);
 	void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *);
 	void (*release)(struct pipe_inode_info *, struct pipe_buffer *);
+	int (*steal)(struct pipe_inode_info *, struct pipe_buffer *);
 };
 
 struct pipe_inode_info {
@@ -52,5 +54,11 @@ void pipe_wait(struct inode * inode);
 
 struct inode* pipe_new(struct inode* inode);
 void free_pipe_info(struct inode* inode);
+
+/*
+ * splice is tied to pipes as a transport (at least for now), so we'll just
+ * add the splice flags here.
+ */
+#define SPLICE_F_MOVE	(0x01)	/* move pages instead of copying */
 
 #endif
Signature · 5 lines
1.3.0.rc1.g384e


-- 
Jens Axboe
#2 lore raw

Re: [PATCH] splice SPLICE_F_MOVE support

Quoted 9 lines · Jens Axboe

Hi,

This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the SPLICE_F_MOVE flag to the splice syscall to ask the kernel to try and move pages, if needed.

Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)

Like Andrew said, you can't check PageLRU without holding zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever.

The following (untested) is something like what I had in mind, and should get stealing closer to working. I've only given it a quick review so far (btw. why do you only unlock the page if it hasn't been stolen?)

With this patch, the ->steal will indicate if the page had been on the LRU or not. If not, then add it; if yes, then do nothing.

There is no caller of ->steal yet that wants the page off the LRU (is there?). That's a bit harder.

---

fs/pipe.c +1 -1
Index: linux-2.6/fs/pipe.c
===================================================================
@@ -124,7 +124,7 @@ static void anon_pipe_buf_unmap(struct p
 static int anon_pipe_buf_steal(struct pipe_inode_info *info,
 			       struct pipe_buffer *buf)
 {
-	buf->stolen = 1;
+	buf->flags |= PIPE_BUF_FLAG_STOLEN;
 	return 0;
 }
 
fs/splice.c +10 -17
Index: linux-2.6/fs/splice.c
===================================================================
@@ -45,17 +45,8 @@ static int page_cache_pipe_buf_steal(str
 	if (!remove_mapping(page_mapping(page), page))
 		return 1;
 
-	if (PageLRU(page)) {
-		struct zone *zone = page_zone(page);
+	buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
 
-		spin_lock_irq(&zone->lru_lock);
-		BUG_ON(!PageLRU(page));
-		__ClearPageLRU(page);
-		del_page_from_lru(zone, page);
-		spin_unlock_irq(&zone->lru_lock);
-	}
-
-	buf->stolen = 1;
 	return 0;
 }
 
@@ -64,7 +55,7 @@ static void page_cache_pipe_buf_release(
 {
 	page_cache_release(buf->page);
 	buf->page = NULL;
-	buf->stolen = 0;
+	buf->flags &= ~(PIPE_BUF_FLAG_STOLEN|PIPE_BUF_FLAG_LRU);
 }
 
 static void *page_cache_pipe_buf_map(struct file *file,
@@ -91,7 +82,7 @@ static void *page_cache_pipe_buf_map(str
 static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info,
 				      struct pipe_buffer *buf)
 {
-	if (!buf->stolen)
+	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN))
 		unlock_page(buf->page);
 	kunmap(buf->page);
 }
@@ -386,11 +377,13 @@ static int pipe_to_file(struct pipe_inod
 	if (sd->flags & SPLICE_F_MOVE) {
 		if (buf->ops->steal(info, buf))
 			goto find_page;
-
 		page = buf->page;
-		if (add_to_page_cache_lru(page, mapping, index,
-						mapping_gfp_mask(mapping)))
+		if (add_to_page_cache(page, mapping, index,
+					mapping_gfp_mask(mapping)))
 			goto find_page;
+
+		if (!(buf->flags & PIPE_BUF_FLAG_LRU))
+			lru_cache_add(page);
 	} else {
 find_page:
 		ret = -ENOMEM;
@@ -435,7 +428,7 @@ find_page:
 	if (ret)
 		goto out;
 
-	if (!buf->stolen) {
+	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
 		char *dst = kmap_atomic(page, KM_USER0);
 
 		memcpy(dst + offset, src + buf->offset, sd->len);
@@ -452,7 +445,7 @@ find_page:
 out:
 	if (ret < 0)
 		unlock_page(page);
-	if (!buf->stolen)
+	if (!(buf->flags & PIPE_BUF_FLAG_STOLEN))
 		page_cache_release(page);
 	buf->ops->unmap(info, buf);
 	return ret;
include/linux/pipe_fs_i.h +4 -1
Index: linux-2.6/include/linux/pipe_fs_i.h
===================================================================
@@ -5,11 +5,14 @@
 
 #define PIPE_BUFFERS (16)
 
+#define PIPE_BUF_FLAG_STOLEN	0x01
+#define PIPE_BUF_FLAG_LRU	0x02
+
 struct pipe_buffer {
 	struct page *page;
 	unsigned int offset, len;
 	struct pipe_buf_operations *ops;
-	unsigned int stolen;
+	unsigned int flags;
 };
 
 struct pipe_buf_operations {
mm/swap.c +1 -1
Index: linux-2.6/mm/swap.c
===================================================================
@@ -177,7 +177,7 @@ void lru_add_drain(void)
 	put_cpu();
 }
 
-#ifdef CONFIG_NUMA
+#ifdef CONFIG_SMP
 static void lru_add_drain_per_cpu(void *dummy)
 {
 	lru_add_drain();
author
#3 lore raw

Re: [PATCH] splice SPLICE_F_MOVE support

Quoted 11 lines · Nick Piggin
Quoted 9 lines · Jens Axboe

Hi,

This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the SPLICE_F_MOVE flag to the splice syscall to ask the kernel to try and move pages, if needed.

Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)

Quoted 8 lines

Like Andrew said, you can't check PageLRU without holding zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever.

The following (untested) is something like what I had in mind, and should get stealing closer to working. I've only given it a quick review so far (btw. why do you only unlock the page if it hasn't been stolen?)

The current branch does not :)

Quoted 5 lines

With this patch, the ->steal will indicate if the page had been on the LRU or not. If not, then add it; if yes, then do nothing.

There is no caller of ->steal yet that wants the page off the LRU (is there?). That's a bit harder.

Thanks Nick, but would you care to rebase it off the 'splice' branch? There's already some changes in this area (notably, getting rid of ->stolen).

git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-2.6-block.git splice

Signature · 1 line
Jens Axboe
#4 lore raw
In reply to Jens Axboe
Quoted 5 lines · Jens Axboe

Hi,

Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.

Haven't looked at this in details, but two small comments already:

- generic_file_splice_read/write should probably go to filemap.c where all the other generic pagecache file operations are
- could we try to replace ->sendfile and ->sendfile with the splice operations completely? Having two different sets of zero-copy file to file transfer mechanisms will make the code pretty messy.

author
#5 lore raw
Quoted 11 lines · Christoph Hellwig
Quoted 5 lines · Jens Axboe

Hi,

Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.

Haven't looked at this in details, but two small comments already:

- generic_file_splice_read/write should probably go to filemap.c where all the other generic pagecache file operations are

Perhaps, I kind of like it local to the splice stuff.

Quoted 3 lines

- could we try to replace ->sendfile and ->sendfile with the splice operations completely? Having two different sets of zero-copy file to file transfer mechanisms will make the code pretty messy.

I've thought of that myself, and yes I hope we can implement ->sendfile() on top of splice if not now then in the immediate future.

Signature · 1 line
Jens Axboe