readv/writev rework

@@ -201,25 +201,29 @@ out:
 }
 
 static ssize_t
-rw_raw_dev(int rw, struct file *filp, char *buf, size_t size, loff_t *offp)
+rw_raw_dev(int rw, struct file *filp, const struct iovec *iov, unsigned long nr_segs, loff_t *offp)
 {
 	const int minor = minor(filp->f_dentry->d_inode->i_rdev);
 	struct block_device *bdev = raw_devices[minor].binding;
 	struct inode *inode = bdev->bd_inode;
+ 	size_t count = iov_length(iov, nr_segs); 
 	ssize_t ret = 0;
 
-	if (size == 0)
-		goto out;
-	ret = -EINVAL;
-	if (size < 0)
-		goto out;
-	ret = -ENXIO;
-	if (*offp >= inode->i_size)
-		goto out;
-
-	if (size + *offp > inode->i_size)
-		size = inode->i_size - *offp;
-	ret = generic_file_direct_IO(rw, inode, buf, *offp, size);
+	if (count == 0)
+		goto out;	
+
+	if ((ssize_t)count < 0)
+		return -EINVAL;	
+
+	if (*offp >= inode->i_size) 
+		return -ENXIO;
+
+	if (count + *offp > inode->i_size) {
+		count = inode->i_size - *offp;
+		nr_segs = iov_shorten((struct iovec *)iov, nr_segs, count);
+	}
+	ret = generic_file_direct_IO(rw, inode, iov, *offp, nr_segs);
+
 	if (ret > 0)
 		*offp += ret;
 out:

@@ -227,15 +231,31 @@ out:
 }
 
 static ssize_t
-raw_read(struct file *filp, char * buf, size_t size, loff_t *offp)
+raw_read(struct file *filp, char *buf, size_t size, loff_t *offp)
 {
-	return rw_raw_dev(READ, filp, buf, size, offp);
+	struct iovec local_iov = { .iov_base = buf, .iov_len = size};
+
+	return rw_raw_dev(READ, filp, &local_iov, 1, offp);
 }
 
 static ssize_t
 raw_write(struct file *filp, const char *buf, size_t size, loff_t *offp)
 {
-	return rw_raw_dev(WRITE, filp, (char *)buf, size, offp);
+	struct iovec local_iov = { .iov_base = buf, .iov_len = size};
+
+	return rw_raw_dev(WRITE, filp, &local_iov, 1, offp);
+}
+
+static ssize_t 
+raw_readv(struct file *filp, const struct iovec *iov, unsigned long nr_segs, loff_t *offp) 
+{
+	return rw_raw_dev(READ, filp, iov, nr_segs, offp);
+}
+
+static ssize_t 
+raw_writev(struct file *filp, const struct iovec *iov, unsigned long nr_segs, loff_t *offp) 
+{
+	return rw_raw_dev(WRITE, filp, iov, nr_segs, offp);
 }
 
 static struct file_operations raw_fops = {

@@ -244,6 +264,8 @@ static struct file_operations raw_fops =
 	.open	=	raw_open,
 	.release=	raw_release,
 	.ioctl	=	raw_ioctl,
+	.readv	= 	raw_readv,
+	.writev	= 	raw_writev,
 	.owner	=	THIS_MODULE,
 };
 

74 lines not on this page · raw message

197 lines not on this page · raw message

@@ -46,6 +46,8 @@ struct file_operations ext2_file_operati
 	.open		= generic_file_open,
 	.release	= ext2_release_file,
 	.fsync		= ext2_sync_file,
+	.readv		= generic_file_readv,
+	.writev		= generic_file_writev,
 	.sendfile	= generic_file_sendfile,
 };
 

@@ -619,11 +619,11 @@ ext2_get_blocks(struct inode *inode, sec
 }
 
 static int
-ext2_direct_IO(int rw, struct inode *inode, char *buf,
-			loff_t offset, size_t count)
+ext2_direct_IO(int rw, struct inode *inode, const struct iovec *iov,
+			loff_t offset, unsigned long nr_segs)
 {
-	return generic_direct_IO(rw, inode, buf,
-				offset, count, ext2_get_blocks);
+	return generic_direct_IO(rw, inode, iov,
+				offset, nr_segs, ext2_get_blocks);
 }
 
 static int

@@ -108,6 +108,8 @@ struct file_operations jfs_file_operatio
 	.write		= generic_file_write,
 	.read		= generic_file_read,
 	.mmap		= generic_file_mmap,
+	.readv		= generic_file_readv,
+	.writev		= generic_file_writev,
  	.sendfile	= generic_file_sendfile,
 	.fsync		= jfs_fsync,
 };

@@ -309,11 +309,11 @@ static int jfs_bmap(struct address_space
 	return generic_block_bmap(mapping, block, jfs_get_block);
 }
 
-static int jfs_direct_IO(int rw, struct inode *inode, char *buf,
-			loff_t offset, size_t count)
+static int jfs_direct_IO(int rw, struct inode *inode, const struct iovec *iov, 
+			loff_t offset, unsigned long nr_segs)
 {
-	return generic_direct_IO(rw, inode, buf,
-				offset, count, jfs_get_blocks);
+	return generic_direct_IO(rw, inode, iov,
+				offset, nr_segs, jfs_get_blocks);
 }
 
 struct address_space_operations jfs_aops = {

201 lines not on this page · raw message

@@ -307,8 +307,7 @@ struct address_space_operations {
 	int (*bmap)(struct address_space *, long);
 	int (*invalidatepage) (struct page *, unsigned long);
 	int (*releasepage) (struct page *, int);
-	int (*direct_IO)(int, struct inode *, char *buf,
-				loff_t offset, size_t count);
+	int (*direct_IO)(int, struct inode *, const struct iovec *iov, loff_t offset, unsigned long nr_segs);
 };
 
 struct backing_dev_info;

@@ -1245,14 +1244,18 @@ extern int generic_file_mmap(struct file
 extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size);
 extern ssize_t generic_file_read(struct file *, char *, size_t, loff_t *);
 extern ssize_t generic_file_write(struct file *, const char *, size_t, loff_t *);
-extern ssize_t generic_file_write_nolock(struct file *, const char *, size_t, loff_t *);
+ssize_t generic_file_write_nolock(struct file *file, const struct iovec *iov,
+				unsigned long nr_segs, loff_t *ppos);
 extern ssize_t generic_file_sendfile(struct file *, struct file *, loff_t *, size_t);
 extern void do_generic_file_read(struct file *, loff_t *, read_descriptor_t *, read_actor_t);
-ssize_t generic_file_direct_IO(int rw, struct inode *inode, char *buf,
-				loff_t offset, size_t count);
-int generic_direct_IO(int rw, struct inode *inode, char *buf,
-			loff_t offset, size_t count, get_blocks_t *get_blocks);
-
+extern ssize_t generic_file_direct_IO(int rw, struct inode *inode, 
+	const struct iovec *iov, loff_t offset, unsigned long nr_segs);
+extern int generic_direct_IO(int rw, struct inode *inode, const struct iovec 
+	*iov, loff_t offset, unsigned long nr_segs, get_blocks_t *get_blocks);
+extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 
+	unsigned long nr_segs, loff_t *ppos);
+ssize_t generic_file_writev(struct file *filp, const struct iovec *iov, 
+			unsigned long nr_segs, loff_t *ppos);
 extern loff_t no_llseek(struct file *file, loff_t offset, int origin);
 extern loff_t generic_file_llseek(struct file *file, loff_t offset, int origin);
 extern loff_t remote_llseek(struct file *file, loff_t offset, int origin);

@@ -42,6 +42,7 @@
 #include <linux/highuid.h>
 #include <linux/brlock.h>
 #include <linux/fs.h>
+#include <linux/uio.h>
 #include <linux/tty.h>
 #include <linux/in6.h>
 #include <linux/completion.h>

@@ -344,6 +345,9 @@ EXPORT_SYMBOL(register_disk);
 EXPORT_SYMBOL(read_dev_sector);
 EXPORT_SYMBOL(init_buffer);
 EXPORT_SYMBOL_GPL(generic_file_direct_IO);
+EXPORT_SYMBOL(generic_file_readv);
+EXPORT_SYMBOL(generic_file_writev);
+EXPORT_SYMBOL(iov_shorten);
 
 /* tty routines */
 EXPORT_SYMBOL(tty_hangup);

211 lines not on this page · raw message

@@ -1399,13 +1399,15 @@ static int ext3_releasepage(struct page 
  * If the O_DIRECT write is intantiating holes inside i_size and the machine
  * crashes then stale disk data _may_ be exposed inside the file.
  */
-static int ext3_direct_IO(int rw, struct inode *inode, char *buf,
-			loff_t offset, size_t count)
+static int ext3_direct_IO(int rw, struct inode *inode,
+			const struct iovec *iov, loff_t offset,
+			unsigned long nr_segs)
 {
 	struct ext3_inode_info *ei = EXT3_I(inode);
 	handle_t *handle = NULL;
 	int ret;
 	int orphan = 0;
+	size_t count = iov_length(iov, nr_segs);
 
 	if (rw == WRITE) {
 		loff_t final_size = offset + count;

@@ -1428,8 +1430,8 @@ static int ext3_direct_IO(int rw, struct
 		}
 	}
 
-	ret = generic_direct_IO(rw, inode, buf, offset,
-				count, ext3_direct_io_get_blocks);
+	ret = generic_direct_IO(rw, inode, iov, offset,
+				nr_segs, ext3_direct_io_get_blocks);
 
 out_stop:
 	if (handle) {

@@ -34,4 +34,19 @@ struct iovec
                                 /* Beg pardon: BSD has 1024 --ANK */
 #endif
 
+/*
+ * Total number of bytes covered by an iovec
+ */
+static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
+{
+	unsigned long seg;
+	size_t ret = 0;
+
+	for (seg = 0; seg < nr_segs; seg++)
+		ret += iov[seg].iov_len;
+	return ret;
+}
+
+unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to);
+
 #endif

@@ -76,19 +76,21 @@ ext3_file_write(struct file *file, const
 }
 
 struct file_operations ext3_file_operations = {
-	.llseek		= generic_file_llseek,	/* BKL held */
-	.read		= generic_file_read,	/* BKL not held.  Don't need */
-	.write		= ext3_file_write,	/* BKL not held.  Don't need */
-	.ioctl		= ext3_ioctl,		/* BKL held */
+	.llseek		= generic_file_llseek,
+	.read		= generic_file_read,
+	.write		= ext3_file_write,
+	.readv		= generic_file_readv,
+	.writev		= generic_file_writev,
+	.ioctl		= ext3_ioctl,
 	.mmap		= generic_file_mmap,
-	.open		= ext3_open_file,		/* BKL not held.  Don't need */
-	.release	= ext3_release_file,	/* BKL not held.  Don't need */
-	.fsync		= ext3_sync_file,		/* BKL held */
-	.sendfile	= generic_file_sendfile,	/* BKL not held.  Don't need */
+	.open		= ext3_open_file,
+	.release	= ext3_release_file,
+	.fsync		= ext3_sync_file,
+	.sendfile	= generic_file_sendfile,
 };
 
 struct inode_operations ext3_file_inode_operations = {
-	.truncate	= ext3_truncate,		/* BKL held */
-	.setattr	= ext3_setattr,		/* BKL held */
+	.truncate	= ext3_truncate,
+	.setattr	= ext3_setattr,
 };
 

@@ -2109,14 +2109,18 @@ generic_file_write_nolock(struct file *f
 		unsigned long index;
 		unsigned long offset;
 		long page_fault;
+		unsigned rest;
+		unsigned copy;
+		unsigned off;
+		struct iovec	*work_iov;
+		char		*work_buf;
+		unsigned	 work_iov_bytes;
 
 		offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
 		index = pos >> PAGE_CACHE_SHIFT;
 		bytes = PAGE_CACHE_SIZE - offset;
 		if (bytes > count)
 			bytes = count;
-		if (bytes + written > iov_bytes)
-			bytes = iov_bytes - written;
 
 		/*
 		 * Bring in the user page that we will copy from _first_.

@@ -2144,7 +2148,29 @@ generic_file_write_nolock(struct file *f
 				vmtruncate(inode, inode->i_size);
 			break;
 		}
-		page_fault = filemap_copy_from_user(page, offset, buf, bytes);
+
+		rest = bytes;
+		off = 0;
+		work_iov = cur_iov;
+		work_buf = buf;
+		work_iov_bytes = iov_bytes;
+		while (rest) {
+			copy = rest;
+			if (written + off == work_iov_bytes) {
+				work_iov++;
+				work_iov_bytes += work_iov->iov_len;
+				work_buf = work_iov->iov_base;
+			}
+			if (copy + written + off > work_iov_bytes)
+				copy = work_iov_bytes - written - off;
+
+			page_fault = filemap_copy_from_user(page, offset+off, work_buf, copy);
+			rest -= copy;
+			off += copy;
+			work_buf += copy;
+			if (unlikely(page_fault))
+				break;
+		}
 		status = a_ops->commit_write(file, page, offset, offset+bytes);
 		if (unlikely(page_fault)) {
 			status = -EFAULT;

@@ -2153,14 +2179,21 @@ generic_file_write_nolock(struct file *f
 				status = bytes;
 
 			if (status >= 0) {
-				written += status;
 				count -= status;
 				pos += status;
-				buf += status;
-				if (written == iov_bytes && count) {
-					cur_iov++;
-					iov_bytes += cur_iov->iov_len;
-					buf = cur_iov->iov_base;
+				rest = status;
+				while (rest) {
+					copy = rest;
+					if (written == iov_bytes) {
+						cur_iov++;
+						iov_bytes += cur_iov->iov_len;
+						buf = cur_iov->iov_base;
+					}
+					if (copy + written > iov_bytes)
+						copy = iov_bytes - written;
+					rest -= copy;
+					written += copy;
+					buf += copy;
 				}
 			}
 		}

@@ -1940,6 +1940,48 @@ filemap_copy_from_user(struct page *page
 	return left;
 }
 
+static inline int
+filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
+			const struct iovec *iov, size_t base, unsigned bytes)
+{
+	char *kaddr;
+	int left = 0;
+
+	kaddr = kmap_atomic(page, KM_USER0);
+	while (bytes) {
+		char *buf = iov->iov_base + base;
+		int copy = min(bytes, iov->iov_len - base);
+		base = 0;
+		if ((left = __copy_from_user(kaddr + offset, buf, copy)))
+			break;
+		bytes -= copy;
+		offset += copy;
+		iov++;
+	}
+	kunmap_atomic(kaddr, KM_USER0);
+	return left;
+}
+
+static inline void
+filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, unsigned bytes)
+{
+	const struct iovec *iov = *iovp;
+	size_t base = *basep;
+
+	while (bytes) {
+		int copy = min(bytes, iov->iov_len - base);
+		bytes -= copy;
+		base += copy;
+		if (iov->iov_len == base) {
+			iov++;
+			base = 0;
+		}
+	}
+	*iovp = iov;
+	*basep = base;
+}
+
+
 /*
  * Write to a file through the page cache. 
  *

@@ -1968,9 +2010,8 @@ generic_file_write_nolock(struct file *f
 	unsigned	bytes;
 	time_t		time_now;
 	struct pagevec	lru_pvec;
-	struct iovec	*cur_iov;
-	unsigned	iov_bytes;	/* Cumulative count to the end of the
-					   current iovec */
+	const struct iovec *cur_iov = iov; /* current iovec */
+	unsigned	iov_base = 0;	   /* offset in the current iovec */
 	unsigned long	seg;
 	char		*buf;
 

@@ -2102,9 +2143,7 @@ generic_file_write_nolock(struct file *f
 		goto out_status;
 	}
 
-	cur_iov = (struct iovec *)iov;
-	iov_bytes = cur_iov->iov_len;
-	buf = cur_iov->iov_base;
+	buf = iov->iov_base;
 	do {
 		unsigned long index;
 		unsigned long offset;

@@ -2115,8 +2154,6 @@ generic_file_write_nolock(struct file *f
 		bytes = PAGE_CACHE_SIZE - offset;
 		if (bytes > count)
 			bytes = count;
-		if (bytes + written > iov_bytes)
-			bytes = iov_bytes - written;
 
 		/*
 		 * Bring in the user page that we will copy from _first_.

@@ -2144,7 +2181,12 @@ generic_file_write_nolock(struct file *f
 				vmtruncate(inode, inode->i_size);
 			break;
 		}
-		page_fault = filemap_copy_from_user(page, offset, buf, bytes);
+
+		if (nr_segs == 1) {
+			page_fault = filemap_copy_from_user(page, offset, buf, bytes);
+		} else {
+			page_fault = filemap_copy_from_user_iovec(page, offset, cur_iov, iov_base, bytes);
+		}
 		status = a_ops->commit_write(file, page, offset, offset+bytes);
 		if (unlikely(page_fault)) {
 			status = -EFAULT;

@@ -2157,11 +2199,8 @@ generic_file_write_nolock(struct file *f
 				count -= status;
 				pos += status;
 				buf += status;
-				if (written == iov_bytes && count) {
-					cur_iov++;
-					iov_bytes += cur_iov->iov_len;
-					buf = cur_iov->iov_base;
-				}
+				if (nr_segs > 1)
+					filemap_set_next_iovec(&cur_iov, &iov_base, status);
 			}
 		}
 		if (!PageReferenced(page))

@@ -1940,6 +1940,63 @@ filemap_copy_from_user(struct page *page
 	return left;
 }
 
+static inline int
+__filemap_copy_from_user_iovec(char *vaddr, 
+			const struct iovec *iov, size_t base, unsigned bytes)
+{
+	int left = 0;
+
+	while (bytes) {
+		char *buf = iov->iov_base + base;
+		int copy = min(bytes, iov->iov_len - base);
+		base = 0;
+		if ((left = __copy_from_user(vaddr, buf, copy)))
+			break;
+		bytes -= copy;
+		vaddr += copy;
+		iov++;
+	}
+	return left;
+}
+
+static inline int
+filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
+			const struct iovec *iov, size_t base, unsigned bytes)
+{
+	char *kaddr;
+	int left;
+
+	kaddr = kmap_atomic(page, KM_USER0);
+	left = __filemap_copy_from_user_iovec(kaddr + offset, iov, base, bytes);
+	kunmap_atomic(kaddr, KM_USER0);
+	if (left != 0) {
+		kaddr = kmap(page);
+		left = __filemap_copy_from_user_iovec(kaddr + offset, iov, base, bytes);
+		kunmap(page);
+	}
+	return left;
+}
+
+static inline void
+filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, unsigned bytes)
+{
+	const struct iovec *iov = *iovp;
+	size_t base = *basep;
+
+	while (bytes) {
+		int copy = min(bytes, iov->iov_len - base);
+		bytes -= copy;
+		base += copy;
+		if (iov->iov_len == base) {
+			iov++;
+			base = 0;
+		}
+	}
+	*iovp = iov;
+	*basep = base;
+}
+
+
 /*
  * Write to a file through the page cache. 
  *

@@ -1968,9 +2025,8 @@ generic_file_write_nolock(struct file *f
 	unsigned	bytes;
 	time_t		time_now;
 	struct pagevec	lru_pvec;
-	struct iovec	*cur_iov;
-	unsigned	iov_bytes;	/* Cumulative count to the end of the
-					   current iovec */
+	const struct iovec *cur_iov = iov; /* current iovec */
+	unsigned	iov_base = 0;	   /* offset in the current iovec */
 	unsigned long	seg;
 	char		*buf;
 

@@ -2102,9 +2158,7 @@ generic_file_write_nolock(struct file *f
 		goto out_status;
 	}
 
-	cur_iov = (struct iovec *)iov;
-	iov_bytes = cur_iov->iov_len;
-	buf = cur_iov->iov_base;
+	buf = iov->iov_base;
 	do {
 		unsigned long index;
 		unsigned long offset;

@@ -2115,8 +2169,6 @@ generic_file_write_nolock(struct file *f
 		bytes = PAGE_CACHE_SIZE - offset;
 		if (bytes > count)
 			bytes = count;
-		if (bytes + written > iov_bytes)
-			bytes = iov_bytes - written;
 
 		/*
 		 * Bring in the user page that we will copy from _first_.

@@ -2144,7 +2196,12 @@ generic_file_write_nolock(struct file *f
 				vmtruncate(inode, inode->i_size);
 			break;
 		}
-		page_fault = filemap_copy_from_user(page, offset, buf, bytes);
+
+		if (nr_segs == 1) {
+			page_fault = filemap_copy_from_user(page, offset, buf, bytes);
+		} else {
+			page_fault = filemap_copy_from_user_iovec(page, offset, cur_iov, iov_base, bytes);
+		}
 		status = a_ops->commit_write(file, page, offset, offset+bytes);
 		if (unlikely(page_fault)) {
 			status = -EFAULT;

@@ -2157,11 +2214,8 @@ generic_file_write_nolock(struct file *f
 				count -= status;
 				pos += status;
 				buf += status;
-				if (written == iov_bytes && count) {
-					cur_iov++;
-					iov_bytes += cur_iov->iov_len;
-					buf = cur_iov->iov_base;
-				}
+				if (nr_segs > 1)
+					filemap_set_next_iovec(&cur_iov, &iov_base, status);
 			}
 		}
 		if (!PageReferenced(page))