splice support #3
Subsystems (6)
PATCH 1/1 splice support #3
Hi,
Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.
----
This adds support for the sys_splice system call. Using a pipe as a transport, it can connect to files or sockets (latter as output only).
arch/i386/kernel/syscall_table.S +1 -0
@@ -312,3 +312,4 @@ ENTRY(sys_call_table) .long sys_unshare /* 310 */ .long sys_set_robust_list .long sys_get_robust_list + .long sys_splice
arch/ia64/kernel/entry.S +1 -0
@@ -1605,5 +1605,6 @@ sys_call_table: data8 sys_ni_syscall // reserved for pselect data8 sys_ni_syscall // 1295 reserved for ppoll data8 sys_unshare + data8 sys_splice .org sys_call_table + 8*NR_syscalls // guard against failures to increase NR_syscalls
fs/Makefile +1 -1
@@ -10,7 +10,7 @@ obj-y := open.o read_write.o file_table. ioctl.o readdir.o select.o fifo.o locks.o dcache.o inode.o \ attr.o bad_inode.o file.o filesystems.o namespace.o aio.o \ seq_file.o xattr.o libfs.o fs-writeback.o mpage.o direct-io.o \ - ioprio.o pnode.o drop_caches.o + ioprio.o pnode.o drop_caches.o splice.o obj-$(CONFIG_INOTIFY) += inotify.o obj-$(CONFIG_EPOLL) += eventpoll.o
fs/ext2/file.c +2 -0
@@ -53,6 +53,8 @@ const struct file_operations ext2_file_o .readv = generic_file_readv, .writev = generic_file_writev, .sendfile = generic_file_sendfile, + .splice_read = generic_file_splice_read, + .splice_write = generic_file_splice_write, };#ifdef CONFIG_EXT2_FS_XIP
fs/ext3/file.c +2 -0
@@ -119,6 +119,8 @@ const struct file_operations ext3_file_o .release = ext3_release_file, .fsync = ext3_sync_file, .sendfile = generic_file_sendfile, + .splice_read = generic_file_splice_read, + .splice_write = generic_file_splice_write, };struct inode_operations ext3_file_inode_operations = {
fs/pipe.c +28 -5
@@ -15,6 +15,7 @@ #include <linux/mount.h>#include <linux/pipe_fs_i.h> #include <linux/uio.h> #include <linux/highmem.h> +#include <linux/pagemap.h> #include <asm/uaccess.h> #include <asm/ioctls.h>
@@ -94,11 +95,20 @@ static void anon_pipe_buf_release(struct {struct page *page = buf->page; -if (info->tmp_page) { -__free_page (page); +/* +* If nobody else uses this page, and we don't already have a +* temporary page, let's keep track of it as a one-deep +* allocation cache +*/ +if (page_count (page) ==1 && !info->tmp_page) { + info->tmp_page = page;return ; } - info->tmp_page = page; + +/* +* Otherwise just release our reference to it +*/ +page_cache_release (page); }static void *anon_pipe_buf_map (struct file *file,struct pipe_inode_info *info,struct pipe_buffer *buf)
@@ -152,6 +162,11 @@ pipe_readv(struct file *filp, const stru chars = total_len; addr = ops->map (filp, info, buf); +if (IS_ERR (addr)) { +if (!ret) + ret =PTR_ERR (addr); +break ; + } error =pipe_iov_copy_to_user (iov, addr + buf->offset, chars); ops->unmap (info, buf);if (unlikely (error)) {
@@ -254,8 +269,16 @@ pipe_writev(struct file *filp, const strstruct pipe_buf_operations *ops = buf->ops;int offset = buf->offset + buf->len;if (ops->can_merge && offset + chars <=PAGE_SIZE ) { -void *addr = ops->map (filp, info, buf); -int error =pipe_iov_copy_from_user (offset + addr, iov, chars); +void *addr; +int error; + + addr = ops->map (filp, info, buf); +if (IS_ERR (addr)) { + error =PTR_ERR (addr); + goto out; + } + error =pipe_iov_copy_from_user (offset + addr, iov, + chars); ops->unmap (info, buf); ret = error; do_wakeup =1 ;
fs/reiserfs/file.c +2 -0
@@ -1576,6 +1576,8 @@ const struct file_operations reiserfs_fi .sendfile = generic_file_sendfile, .aio_read = generic_file_aio_read, .aio_write = reiserfs_aio_write, + .splice_read = generic_file_splice_read, + .splice_write = generic_file_splice_write, };struct inode_operations reiserfs_file_inode_operations = {
fs/splice.c new file +612 -0
612 lines not on this page · raw message
include/asm-i386/unistd.h +2 -1
@@ -318,8 +318,9 @@ #define __NR_ppoll 309#define __NR_unshare 310#define __NR_set_robust_list 311#define __NR_get_robust_list 312 +#define __NR_sys_splice 313 -#define NR_syscalls 313 +#define NR_syscalls 314/* * user-visible error numbers are in the range -1 - -128: see
include/asm-ia64/unistd.h +2 -1
@@ -285,12 +285,13 @@ #define __NR_fchmodat 1292#define __NR_faccessat 1293/* 1294, 1295 reserved for pselect/ppoll */ #define __NR_unshare 1296 +#define __NR_splice 1297#ifdef __KERNEL__#include <linux/config.h> -#define NR_syscalls 273/* length of syscall table */ +#define NR_syscalls 274/* length of syscall table */ #define __ARCH_WANT_SYS_RT_SIGACTION
include/asm-powerpc/unistd.h +2 -1
@@ -301,8 +301,9 @@ #define __NR_spu_create 279#define __NR_pselect6 280#define __NR_ppoll 281#define __NR_unshare 282 +#define __NR_splice 283 -#define __NR_syscalls 283 +#define __NR_syscalls 284#ifdef __KERNEL__#define __NR__exit __NR_exit
include/asm-x86_64/unistd.h +3 -1
@@ -609,8 +609,10 @@ #define __NR_set_robust_list 273 __SYSCALL(__NR_set_robust_list, sys_set_robust_list)#define __NR_get_robust_list 274 __SYSCALL(__NR_get_robust_list, sys_get_robust_list) +#define __NR_splice 275 +__SYSCALL(__NR_splice, sys_splice) -#define __NR_syscall_max __NR_get_robust_list +#define __NR_syscall_max __NR_splice#ifndef __NO_STUBS
include/linux/fs.h +4 -0
@@ -1032,6 +1032,8 @@ struct file_operations {int (*check_flags)(int );int (*dir_notify)(struct file *filp,unsigned long arg);int (*flock) (struct file *,int ,struct file_lock *); +ssize_t (*splice_write)(struct inode *,struct file *,size_t ,unsigned int ); +ssize_t (*splice_read)(struct file *,struct inode *,size_t ,unsigned int ); };struct inode_operations {
@@ -1609,6 +1611,8 @@ extern ssize_t generic_file_sendfile(strextern void do_generic_mapping_read (struct address_space *mapping,struct file_ra_state *,struct file *,loff_t *,read_descriptor_t *,read_actor_t ); +extern ssize_t generic_file_splice_read (struct file *,struct inode *,size_t ,unsigned int ); +extern ssize_t generic_file_splice_write (struct inode *,struct file *,size_t ,unsigned int );extern void file_ra_state_init (struct file_ra_state *ra,struct address_space *mapping); extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
include/linux/syscalls.h +2 -0
@@ -569,5 +569,7 @@ asmlinkage long compat_sys_newfstatat(unasmlinkage long compat_sys_openat (unsigned int dfd,const char __user *filename,int flags,int mode);asmlinkage long sys_unshare (unsigned long unshare_flags); +asmlinkage long sys_splice (int fdin,int fdout,size_t len, +unsigned int flags);#endif
net/socket.c +5 -1
@@ -119,7 +119,10 @@ static ssize_t sock_writev(struct file *static ssize_t sock_sendpage (struct file *file,struct page *page,int offset,size_t size,loff_t *ppos,int more); +extern ssize_t generic_splice_sendpage (struct inode *inode,struct file *out, +size_t len,unsigned int flags); +/* * Socket files have a set of 'special' operations as well as the generic file ones. These don't appear * in the operation structures but are done directly via the socketcall() multiplexor.
@@ -141,7 +144,8 @@ #endif .fasync = sock_fasync, .readv = sock_readv, .writev = sock_writev, - .sendpage = sock_sendpage + .sendpage = sock_sendpage, + .splice_write = generic_splice_sendpage, };/*
Signature · 1 line
Jens Axboe
PATCH 1/1 splice SPLICE_F_MOVE support
Hi,
This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the SPLICE_F_MOVE flag to the splice syscall to ask the kernel to try and move pages, if needed.
Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)
---
From: Jens Axboe <[email protected]>
Date: Thu Mar 30 15:16:46 2006 +0200
Subject: [PATCH] splice: add support for SPLICE_F_MOVE flag
This enables the caller to migrate pages from one address space page cache to another. In buzz word marketing, you can do zero-copy file copies!
---
fs/pipe.c | 8 +++ fs/splice.c | 121 +++++++++++++++++++++++++++++++-------------- include/linux/pipe_fs_i.h | 8 +++ 3 files changed, 100 insertions(+), 37 deletions(-)
54facc65189d86f4b8b7eb9be5c0c5623090483d
fs/pipe.c +8 -0
@@ -121,11 +121,19 @@ static void anon_pipe_buf_unmap(struct pkunmap (buf->page); } +static int anon_pipe_buf_steal (struct pipe_inode_info *info, +struct pipe_buffer *buf) +{ + buf->stolen =1 ; +return 0 ; +} +static struct pipe_buf_operations anon_pipe_buf_ops = { .can_merge =1 , .map = anon_pipe_buf_map, .unmap = anon_pipe_buf_unmap, .release = anon_pipe_buf_release, + .steal = anon_pipe_buf_steal, }; static ssize_t
fs/splice.c +84 -37
179 lines not on this page · raw message
include/linux/pipe_fs_i.h +8 -0
@@ -9,6 +9,7 @@ struct pipe_buffer {struct page *page;unsigned int offset, len;struct pipe_buf_operations *ops; +unsigned int stolen; };struct pipe_buf_operations {
@@ -16,6 +17,7 @@ struct pipe_buf_operations {void * (*map)(struct file *,struct pipe_inode_info *,struct pipe_buffer *);void (*unmap)(struct pipe_inode_info *,struct pipe_buffer *);void (*release)(struct pipe_inode_info *,struct pipe_buffer *); +int (*steal)(struct pipe_inode_info *,struct pipe_buffer *); };struct pipe_inode_info {
@@ -52,5 +54,11 @@ void pipe_wait(struct inode * inode);struct inode *pipe_new (struct inode * inode);void free_pipe_info (struct inode * inode); + +/* +* splice is tied to pipes as a transport (at least for now), so we'll just +* add the splice flags here. +*/ +#define SPLICE_F_MOVE (0x01)/* move pages instead of copying */ #endif
Signature · 5 lines
1.3.0.rc1.g384e -- Jens Axboe
Re: [PATCH] splice SPLICE_F_MOVE support
Quoted 9 lines · Jens Axboe
Hi,
This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the
SPLICE_F_MOVEflag to the splice syscall to ask the kernel to try and move pages, if needed.Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)
Like Andrew said, you can't check PageLRU without holding zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever.
The following (untested) is something like what I had in mind, and should get stealing closer to working. I've only given it a quick review so far (btw. why do you only unlock the page if it hasn't been stolen?)
With this patch, the ->steal will indicate if the page had been on the LRU or not. If not, then add it; if yes, then do nothing.
There is no caller of ->steal yet that wants the page off the LRU (is there?). That's a bit harder.
---
fs/pipe.c +1 -1
@@ -124,7 +124,7 @@ static void anon_pipe_buf_unmap(struct pstatic int anon_pipe_buf_steal (struct pipe_inode_info *info,struct pipe_buffer *buf) { - buf->stolen =1 ; + buf->flags |=PIPE_BUF_FLAG_STOLEN ;return 0 ; }
fs/splice.c +10 -17
@@ -45,17 +45,8 @@ static int page_cache_pipe_buf_steal(strif (!remove_mapping (page_mapping (page), page))return 1 ; -if (PageLRU (page)) { -struct zone *zone =page_zone (page); + buf->flags |=PIPE_BUF_FLAG_STOLEN |PIPE_BUF_FLAG_LRU ; -spin_lock_irq (&zone->lru_lock); -BUG_ON (!PageLRU (page)); -__ClearPageLRU (page); -del_page_from_lru (zone, page); -spin_unlock_irq (&zone->lru_lock); - } - - buf->stolen =1 ;return 0 ; }
@@ -64,7 +55,7 @@ static void page_cache_pipe_buf_release( {page_cache_release (buf->page); buf->page =NULL ; - buf->stolen =0 ; + buf->flags &= ~(PIPE_BUF_FLAG_STOLEN |PIPE_BUF_FLAG_LRU ); } static void *page_cache_pipe_buf_map(struct file *file,
@@ -91,7 +82,7 @@ static void *page_cache_pipe_buf_map(strstatic void page_cache_pipe_buf_unmap (struct pipe_inode_info *info,struct pipe_buffer *buf) { -if (!buf->stolen) +if (!(buf->flags &PIPE_BUF_FLAG_STOLEN ))unlock_page (buf->page);kunmap (buf->page); }
@@ -386,11 +377,13 @@ static int pipe_to_file(struct pipe_inodif (sd->flags &SPLICE_F_MOVE ) {if (buf->ops->steal (info, buf)) goto find_page; - page = buf->page; -if (add_to_page_cache_lru (page, mapping, index, -mapping_gfp_mask (mapping))) +if (add_to_page_cache (page, mapping, index, +mapping_gfp_mask (mapping))) goto find_page; + +if (!(buf->flags &PIPE_BUF_FLAG_LRU )) +lru_cache_add (page); }else { find_page: ret = -ENOMEM ;
@@ -435,7 +428,7 @@ find_page:if (ret) goto out; -if (!buf->stolen) { +if (!(buf->flags &PIPE_BUF_FLAG_STOLEN )) {char *dst =kmap_atomic (page,KM_USER0 );memcpy (dst + offset, src + buf->offset, sd->len);
@@ -452,7 +445,7 @@ find_page: out:if (ret <0 )unlock_page (page); -if (!buf->stolen) +if (!(buf->flags &PIPE_BUF_FLAG_STOLEN ))page_cache_release (page); buf->ops->unmap (info, buf);return ret;
include/linux/pipe_fs_i.h +4 -1
@@ -5,11 +5,14 @@#define PIPE_BUFFERS (16) +#define PIPE_BUF_FLAG_STOLEN 0x01 +#define PIPE_BUF_FLAG_LRU 0x02 +struct pipe_buffer {struct page *page;unsigned int offset, len;struct pipe_buf_operations *ops; -unsigned int stolen; +unsigned int flags; };struct pipe_buf_operations {
mm/swap.c +1 -1
@@ -177,7 +177,7 @@ void lru_add_drain(void)put_cpu (); } -#ifdef CONFIG_NUMA +#ifdef CONFIG_SMP static void lru_add_drain_per_cpu (void *dummy) {lru_add_drain ();
Re: [PATCH] splice SPLICE_F_MOVE support
Quoted 11 lines · Nick Piggin
Quoted 9 lines · Jens Axboe
Hi,
This applies on top of the splice #3 just posted, adding support for moving of pages. The caller can use the
SPLICE_F_MOVEflag to the splice syscall to ask the kernel to try and move pages, if needed.Disclaimer: this works for me, but may have vm issues that I missed. CC'ing Nick :-)
Quoted 8 lines
Like Andrew said, you can't check PageLRU without holding
zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever.The following (untested) is something like what I had in mind, and should get stealing closer to working. I've only given it a quick review so far (btw. why do you only unlock the page if it hasn't been stolen?)
The current branch does not :)
Quoted 5 lines
With this patch, the ->steal will indicate if the page had been on the LRU or not. If not, then add it; if yes, then do nothing.
There is no caller of ->steal yet that wants the page off the LRU (is there?). That's a bit harder.
Thanks Nick, but would you care to rebase it off the 'splice' branch? There's already some changes in this area (notably, getting rid of ->stolen).
git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-2.6-block.git splice
Signature · 1 line
Jens Axboe
Quoted 5 lines · Jens Axboe
Hi,
Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.
Haven't looked at this in details, but two small comments already:
- generic_file_splice_read/write should probably go to filemap.c where all the other generic pagecache file operations are
- could we try to replace ->sendfile and ->sendfile with the splice operations completely? Having two different sets of zero-copy file to file transfer mechanisms will make the code pretty messy.
Quoted 11 lines · Christoph Hellwig
Quoted 5 lines · Jens Axboe
Hi,
Ok, this should be it, I hope. Fixed the remaining issues spotted by akpm, and also thanks to KAMEZAWA Hiroyuki for pointing out that the page moving logic could get confused.
Haven't looked at this in details, but two small comments already:
- generic_file_splice_read/write should probably go to
filemap.cwhere all the other generic pagecache file operations are
Perhaps, I kind of like it local to the splice stuff.
Quoted 3 lines
- could we try to replace ->sendfile and ->sendfile with the splice operations completely? Having two different sets of zero-copy file to file transfer mechanisms will make the code pretty messy.
I've thought of that myself, and yes I hope we can implement ->sendfile() on top of splice if not now then in the immediate future.
Signature · 1 line
Jens Axboe