RFC: exporting per-superblock statistics to user space
Subsystems (3)
we still have a need to provide "iostat" like statistics for NFS clients. attached are a couple of patches, against 2.6.11.3, which prototype an approach for providing this kind of data to user programs.
i'd like some comment on the approach.
01-mountstats.patch adds a new file called /proc/self/mountstats and a new file system hook called show_stats. this just replicates /proc/mounts and the show_options hook.
02-nfs-iostat.patch teachs the NFS client to use the new show_stats hook as a demonstration.
note that this approach addresses previously voiced concerns about exporting per-superblock stats to user space.
1. processes can't see stats for file systems mounted outside their namespace.
2. reading the stats file is serialized with mount and unmount operations.
3. the approach doesn't use /sys or kobjects.
4. there are no lifetime issues tied to file systems loaded as a module.
[PATCH] VFS: New /proc file /proc/self/mountstats
Create a new file under /proc/self, called mountstats, where mounted file systems can export information (configuration options, performance counters, and so on). Use a mechanism similar to /proc/mounts and s_ops->show_options.
This mechanism does not violate namespace security, and is safe to use while other processes are unmounting file systems.
Test-plan:
Test concurrent mount/unmount operations while cat'ing /proc/self/mountstats.
Version: Mon, 14 Mar 2005 17:06:04 -0500
---
fs/namespace.c | 66 +++++++++++++++++++++++++++++++++++++++++++++ fs/proc/base.c | 40 +++++++++++++++++++++++++++ include/linux/fs.h | 1 3 files changed, 107 insertions(+)
fs/namespace.c +66 -0
@@ -265,6 +265,72 @@ struct seq_operations mounts_op = { .show = show_vfsmnt }; +/* iterator */ +static void *ms_start (struct seq_file *m,loff_t *pos) +{ +struct namespace *n = m->private; +struct list_head *p; +loff_t l = *pos; + +down_read (&n->sem); +list_for_each (p, &n->list) +if (!l--) + return list_entry(p, struct vfsmount, mnt_list); +return NULL ; +} + +static void *ms_next (struct seq_file *m,void *v,loff_t *pos) +{ +struct namespace *n = m->private; +struct list_head *p = ((struct vfsmount *)v)->mnt_list.next; + (*pos)++; + return p==&n->list ? NULL : list_entry(p, struct vfsmount, mnt_list); +} + +static void ms_stop (struct seq_file *m,void *v) +{ +struct namespace *n = m->private; +up_read (&n->sem); +} + +static int show_vfsstat (struct seq_file *m,void *v) +{ +struct vfsmount *mnt = v; +int err =0 ; + +/* device */ +if (mnt->mnt_devname) { +seq_puts (m,"device " ); +mangle (m, mnt->mnt_devname); + }else +seq_puts (m,"no device" ); + +/* mount point */ +seq_puts (m," mounted on " ); +seq_path (m, mnt, mnt->mnt_root," \t\n\\" ); +seq_putc (m,' ' ); + +/* file system type */ +seq_puts (m,"with fstype " ); +mangle (m, mnt->mnt_sb->s_type->name); + +/* optional statistics */ +if (mnt->mnt_sb->s_op->show_stats) { +seq_putc (m,' ' ); + err = mnt->mnt_sb->s_op->show_stats (m, mnt); + } + +seq_putc (m,'\n' ); +return err; +} + +struct seq_operations mountstats_op = { + .start = ms_start, + .next = ms_next, + .stop = ms_stop, + .show = show_vfsstat, +}; +/** * may_umount_tree - check if a mount tree is busy * @mnt: root of mount tree
fs/proc/base.c +40 -0
@@ -60,6 +60,7 @@ enum pid_directory_inos {PROC_TGID_STATM ,PROC_TGID_MAPS ,PROC_TGID_MOUNTS , +PROC_TGID_MOUNTSTATS ,PROC_TGID_WCHAN , #ifdef CONFIG_SCHEDSTATS PROC_TGID_SCHEDSTAT,
@@ -91,6 +92,7 @@ enum pid_directory_inos {PROC_TID_STATM ,PROC_TID_MAPS ,PROC_TID_MOUNTS , +PROC_TID_MOUNTSTATS ,PROC_TID_WCHAN , #ifdef CONFIG_SCHEDSTATS PROC_TID_SCHEDSTAT,
@@ -134,6 +136,7 @@ static struct pid_entry tgid_base_stuff[E (PROC_TGID_ROOT ,"root" ,S_IFLNK |S_IRWXUGO ),E (PROC_TGID_EXE ,"exe" ,S_IFLNK |S_IRWXUGO ),E (PROC_TGID_MOUNTS ,"mounts" ,S_IFREG |S_IRUGO ), +E (PROC_TGID_MOUNTSTATS ,"mountstats" ,S_IFREG |S_IRUGO ), #ifdef CONFIG_SECURITY E(PROC_TGID_ATTR, "attr", S_IFDIR|S_IRUGO|S_IXUGO), #endif
@@ -164,6 +167,7 @@ static struct pid_entry tid_base_stuff[]E (PROC_TID_ROOT ,"root" ,S_IFLNK |S_IRWXUGO ),E (PROC_TID_EXE ,"exe" ,S_IFLNK |S_IRWXUGO ),E (PROC_TID_MOUNTS ,"mounts" ,S_IFREG |S_IRUGO ), +E (PROC_TID_MOUNTSTATS ,"mountstats" ,S_IFREG |S_IRUGO ), #ifdef CONFIG_SECURITY E(PROC_TID_ATTR, "attr", S_IFDIR|S_IRUGO|S_IXUGO), #endif
@@ -528,6 +532,38 @@ static struct file_operations proc_mount .release = mounts_release, }; +extern struct seq_operations mountstats_op; +static int mountstats_open (struct inode *inode,struct file *file) +{ +struct task_struct *task =proc_task (inode); +int ret =seq_open (file, &mountstats_op); + +if (!ret) { +struct seq_file *m = file->private_data; +struct namespace *namespace; +task_lock (task); + namespace = task->namespace; +if (namespace) +get_namespace (namespace); +task_unlock (task); + +if (namespace) + m->private = namespace; +else { +seq_release (inode, file); + ret = -EINVAL ; + } + } +return ret; +} + +static struct file_operations proc_mountstats_operations = { + .open = mountstats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = mounts_release, +}; +#define PROC_BLOCK_SIZE (3*1024)/* 4K page size but our output routines use some slack for overruns */ static ssize_t proc_info_read(struct file * file, char __user * buf,
@@ -1447,6 +1483,10 @@ static struct dentry *proc_pident_lookupcase PROC_TGID_MOUNTS : inode->i_fop = &proc_mounts_operations;break ; +case PROC_TID_MOUNTSTATS : +case PROC_TGID_MOUNTSTATS : + inode->i_fop = &proc_mountstats_operations; +break ;#ifdef CONFIG_SECURITY case PROC_TID_ATTR : inode->i_nlink =2 ;
include/linux/fs.h +1 -0
@@ -1008,6 +1008,7 @@ struct super_operations {void (*umount_begin) (struct super_block *);int (*show_options)(struct seq_file *,struct vfsmount *); +int (*show_stats)(struct seq_file *,struct vfsmount *);ssize_t (*quota_read)(struct super_block *,int ,char *,size_t ,loff_t );ssize_t (*quota_write)(struct super_block *,int ,const char *,size_t ,loff_t );
[PATCH] NFS: add I/O performance counters
Add an extensible per-superblock performance counter facility to the NFS client. This facility mimics the counters available for block devices and for networking.
Expose these new counters via /proc/self/mountstats.
Test-plan:
fsx and iozone on SMP systems. Watch for memory overwrite bugs, and performance loss (significantly more CPU required per op).
Version: Mon, 14 Mar 2005 17:06:12 -0500
---
fs/nfs/dir.c | 8 ++ fs/nfs/direct.c | 5 + fs/nfs/file.c | 20 +++-- fs/nfs/inode.c | 126 +++++++++++++++++++++++++++++++++++-- fs/nfs/pagelist.c | 12 ++- fs/nfs/read.c | 7 ++ fs/nfs/write.c | 10 ++ include/linux/nfs_fs_sb.h | 5 + include/linux/nfs_iostat.h | 80 +++++++++++++++++++++++ 9 files changed, 256 insertions(+), 17 deletions(-)
fs/nfs/dir.c +8 -0
@@ -27,6 +27,7 @@#include <linux/mm.h> #include <linux/sunrpc/clnt.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_mount.h> #include <linux/pagemap.h> #include <linux/smp_lock.h>
@@ -428,6 +429,8 @@ static int nfs_readdir(struct file *filplock_kernel (); +nfs_inc_stats (inode,NFS_VFS_GETDENTS ); + res =nfs_revalidate_inode (NFS_SERVER (inode), inode);if (res <0 ) {unlock_kernel ();
@@ -584,6 +587,7 @@ static int nfs_lookup_revalidate(struct parent =dget_parent (dentry);lock_kernel (); dir = parent->d_inode; +nfs_inc_stats (dir,NFS_DENTRY_REVALIDATE ); inode = dentry->d_inode;if (nd && !(nd->flags &LOOKUP_CONTINUE ) && (nd->flags &LOOKUP_OPEN ))
@@ -712,6 +716,7 @@ static struct dentry *nfs_lookup(structdfprintk (VFS ,"NFS: lookup(%s/%s)\n" , dentry->d_parent->d_name.name, dentry->d_name.name); +nfs_inc_stats (dir,NFS_VFS_LOOKUP ); res =ERR_PTR (-ENAMETOOLONG );if (dentry->d_name.len >NFS_SERVER (dir)->namelen)
@@ -1116,6 +1121,7 @@ static int nfs_sillyrename(struct inodedfprintk (VFS ,"NFS: silly-rename(%s/%s, ct=%d)\n" , dentry->d_parent->d_name.name, dentry->d_name.name,atomic_read (&dentry->d_count)); +nfs_inc_stats (dir,NFS_SILLY_RENAME );#ifdef NFS_PARANOIA if (!dentry->d_inode)
@@ -1500,6 +1506,8 @@ int nfs_permission(struct inode *inode,struct rpc_cred *cred;int res; +nfs_inc_stats (inode,NFS_VFS_ACCESS ); +if (mask ==0 )return 0 ;
fs/nfs/direct.c +5 -0
@@ -47,6 +47,7 @@#include <linux/kref.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_page.h> #include <linux/sunrpc/clnt.h>
@@ -354,6 +355,8 @@ static ssize_t nfs_direct_read_seg(struc result =nfs_direct_read_wait (dreq, clnt->cl_intr);rpc_clnt_sigunmask (clnt, &oldset); +nfs_add_stats (inode,NFS_WIRE_READ_BYTES , result); +nfs_add_stats (inode,NFS_DIRECT_READ_BYTES , result);return result; }
@@ -576,6 +579,8 @@ static ssize_t nfs_direct_write(struct iif (result < size)break ; } +nfs_add_stats (inode,NFS_WIRE_WRITTEN_BYTES , tot_bytes); +nfs_add_stats (inode,NFS_DIRECT_WRITTEN_BYTES , tot_bytes);return tot_bytes; }
fs/nfs/file.c +12 -8
@@ -22,6 +22,7 @@#include <linux/fcntl.h> #include <linux/stat.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_mount.h> #include <linux/mm.h> #include <linux/slab.h>
@@ -86,18 +87,15 @@ static int nfs_check_flags(int flags)static int nfs_file_open (struct inode *inode,struct file *filp) { -struct nfs_server *server =NFS_SERVER (inode); -int (*open)(struct inode *,struct file *);int res; res =nfs_check_flags (filp->f_flags);if (res)return res; +nfs_inc_stats (inode,NFS_VFS_OPEN );lock_kernel (); -/* Do NFSv4 open() call */ -if ((open = server->rpc_ops->file_open) !=NULL ) - res =open (inode, filp); + res =NFS_SERVER (inode)->rpc_ops->file_open (inode, filp);unlock_kernel ();return res; }
@@ -105,6 +103,7 @@ nfs_file_open(struct inode *inode, strucstatic int nfs_file_release (struct inode *inode,struct file *filp) { +nfs_inc_stats (inode,NFS_VFS_CLOSE );return NFS_PROTO (inode)->file_release (inode, filp); }
@@ -123,6 +122,7 @@ nfs_file_flush(struct file *file)if ((file->f_mode &FMODE_WRITE ) ==0 )return 0 ; +nfs_inc_stats (inode,NFS_VFS_FLUSH );lock_kernel ();/* Ensure that data+attribute caches are up to date after close() */ status =nfs_wb_all (inode);
@@ -155,6 +155,7 @@ nfs_file_read(struct kiocb *iocb, char _ result =nfs_revalidate_inode (NFS_SERVER (inode), inode);if (!result) result =generic_file_aio_read (iocb, buf, count, pos); +nfs_add_stats (inode,NFS_SYS_READ_BYTES , result);return result; }
@@ -206,6 +207,7 @@ nfs_fsync(struct file *file, struct dentdfprintk (VFS ,"nfs: fsync(%s/%ld)\n" , inode->i_sb->s_id, inode->i_ino); +nfs_inc_stats (inode,NFS_VFS_FSYNC );lock_kernel (); status =nfs_wb_all (inode);if (!status) {
@@ -284,6 +286,7 @@ nfs_file_write(struct kiocb *iocb, const goto out; result =generic_file_aio_write (iocb, buf, count, pos); +nfs_add_stats (inode,NFS_SYS_WRITTEN_BYTES , result); out:return result;
@@ -406,13 +409,14 @@ nfs_lock(struct file *filp, int cmd, str {struct inode * inode = filp->f_mapping->host; +if (!inode) +return -EINVAL ; +dprintk ("NFS: nfs_lock(f=%s/%ld, t=%x, fl=%x, r=%Ld:%Ld)\n" , inode->i_sb->s_id, inode->i_ino, fl->fl_type, fl->fl_flags, (long long )fl->fl_start, (long long )fl->fl_end); - -if (!inode) -return -EINVAL ; +nfs_inc_stats (inode,NFS_VFS_LOCK );/* No mandatory locks over NFS */ if ((inode->i_mode & (S_ISGID |S_IXGRP )) ==S_ISGID )
fs/nfs/inode.c +120 -6
212 lines not on this page · raw message
fs/nfs/pagelist.c +9 -3
@@ -17,6 +17,7 @@#include <linux/nfs4.h> #include <linux/nfs_page.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_mount.h> #define NFS_PARANOIA 1
@@ -192,10 +193,15 @@ nfs_wait_on_request(struct nfs_page *req {struct inode *inode = req->wb_context->dentry->d_inode;struct rpc_clnt *clnt =NFS_CLIENT (inode); +unsigned long start = jiffies; +int result =0 ; -if (!NFS_WBACK_BUSY (req)) -return 0 ; -return nfs_wait_event (clnt, req->wb_context->waitq, !NFS_WBACK_BUSY (req)); +if (NFS_WBACK_BUSY (req)) { + result =nfs_wait_event (clnt, req->wb_context->waitq, !NFS_WBACK_BUSY (req)); +nfs_add_stats (inode,NFS_WAIT_EVENT_JIFFIES , (jiffies - start)); +nfs_inc_stats (inode,NFS_WAIT_EVENT ); + } +return result; }/**
fs/nfs/read.c +7 -0
@@ -26,6 +26,7 @@#include <linux/pagemap.h> #include <linux/sunrpc/clnt.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_page.h> #include <linux/smp_lock.h>
@@ -134,6 +135,7 @@ static int nfs_readpage_sync(struct nfs_ } count -= result; rdata->args.pgbase += result; +nfs_add_stats (inode,NFS_WIRE_READ_BYTES , result);/* Note: result == 0 should only happen if we're caching * a write that extends the file and punches a hole. */
@@ -464,6 +466,7 @@ void nfs_readpage_result(struct rpc_task/* Is this a short read? */ if (task->tk_status >=0 && resp->count < argp->count && !resp->eof) { +nfs_inc_stats (data->inode,NFS_SHORT_READ );/* Has the server at least made some progress? */ if (resp->count !=0 ) {/* Yes, so retry the read at the end of the data */
@@ -477,6 +480,7 @@ void nfs_readpage_result(struct rpc_task }NFS_FLAGS (data->inode) |=NFS_INO_INVALID_ATIME ; data->complete (data, status); +nfs_add_stats (data->inode,NFS_WIRE_READ_BYTES , resp->count); }/*
@@ -493,6 +497,8 @@ int nfs_readpage(struct file *file, strudprintk ("NFS: nfs_readpage (%p %ld@%lu)\n" , page,PAGE_CACHE_SIZE , page->index); +nfs_inc_stats (inode,NFS_VFS_READPAGE ); +/* * Try to flush any pending writes to the file.. *
@@ -573,6 +579,7 @@ int nfs_readpages(struct file *filp, str inode->i_sb->s_id, (long long )NFS_FILEID (inode), nr_pages); +nfs_inc_stats (inode,NFS_VFS_READPAGES );if (filp ==NULL ) { desc.ctx =nfs_find_open_context (inode,FMODE_READ );
fs/nfs/write.c +10 -0
@@ -57,6 +57,7 @@#include <linux/sunrpc/clnt.h> #include <linux/nfs_fs.h> +#include <linux/nfs_iostat.h> #include <linux/nfs_mount.h> #include <linux/nfs_page.h> #include <asm/uaccess.h>
@@ -105,6 +106,7 @@ static void nfs_grow_file(struct page *p end = ((loff_t )page->index <<PAGE_CACHE_SHIFT ) + ((loff_t )offset+count);if (i_size >= end)return ; +nfs_inc_stats (inode,NFS_EXTEND_WRITE );i_size_write (inode, end); }
@@ -193,6 +195,7 @@ static int nfs_writepage_sync(struct nfs wdata->args.pgbase += result; written += result; count -= result; +nfs_add_stats (inode,NFS_WIRE_WRITTEN_BYTES , result); }while (count);/* Update file length */ nfs_grow_file (page, offset, written);
@@ -251,6 +254,8 @@ int nfs_writepage(struct page *page, strint priority =wb_priority (wbc);int err; +nfs_inc_stats (inode,NFS_VFS_WRITEPAGE ); +/* * Note: We need to ensure that we have a reference to the inode * if we are to do asynchronous writes. If not, waiting
@@ -318,6 +323,8 @@ int nfs_writepages(struct address_spacestruct inode *inode = mapping->host;int err; +nfs_inc_stats (inode,NFS_VFS_WRITEPAGES ); + err =generic_writepages (mapping, wbc);if (err)return err;
@@ -1145,6 +1152,8 @@ void nfs_writeback_done(struct rpc_taskif (task->tk_status >=0 && resp->count < argp->count) {static unsigned long complain; +nfs_inc_stats (data->inode,NFS_SHORT_WRITE ); +/* Has the server at least made some progress? */ if (resp->count !=0 ) {/* Was this an NFSv2 write or an NFSv3 stable write? */
@@ -1176,6 +1185,7 @@ void nfs_writeback_done(struct rpc_task* Process the nfs_page list */ data->complete (data, task->tk_status); +nfs_add_stats (data->inode,NFS_WIRE_WRITTEN_BYTES , resp->count); }
include/linux/nfs_fs_sb.h +5 -0
@@ -4,6 +4,8 @@#include <linux/list.h> #include <linux/backing-dev.h> +struct nfs_iostats ; +/* * NFS client parameters stored in the superblock. */
@@ -11,6 +13,7 @@ struct nfs_server {struct rpc_clnt * client;/* RPC client handle */ struct rpc_clnt * client_sys;/* 2nd handle for FSINFO */ struct nfs_rpc_ops * rpc_ops;/* NFS protocol vector */ +struct nfs_iostats * io_stats;/* I/O statistics */ struct backing_dev_info backing_dev_info;int flags;/* various flags */ unsigned int caps;/* server capabilities */
@@ -25,6 +28,8 @@ struct nfs_server {unsigned int acregmax;unsigned int acdirmin;unsigned int acdirmax; +unsigned long retrans_timeo;/* retransmit timeout */ +unsigned int retrans_count;/* number of retransmit tries */ unsigned int namelen;char * hostname;/* remote hostname */ struct nfs_fh fh;
include/linux/nfs_iostat.h +80 -0
80 lines not on this page · raw message
begin:vcard
fn:Chuck Lever
n:Lever;Charles
org:Network Appliance, Incorporated;Linux NFS Client Development
adr:535 West William Street, Suite 3100;;Center for Information Technology Integration;Ann Arbor;MI;48103-4943;USA email;internet:[email protected]
title:Member of Technical Staff
tel;work:+1 734 763 4415
tel;fax:+1 734 763 4434
tel;home:+1 734 668 1089
x-mozilla-html:FALSE
url:http://www.monkey.org/~cel/
version:2.1
end:vcard
Quoted 8 lines · Chuck Lever
we still have a need to provide "iostat" like statistics for NFS clients. attached are a couple of patches, against 2.6.11.3, which prototype an approach for providing this kind of data to user programs.
i'd like some comment on the approach.01-mountstats.patch adds a new file called
/proc/self/mountstatsand a new file system hook calledshow_stats. this just replicates/proc/mountsand theshow_optionshook.
I don't think this makes a lot of sense. The stats aren't per namespace or mountpoint but per fs-instance, right?
on den 23.03.2005 Klokka 11:06 (+0000) skreiv Christoph Hellwig:
Quoted 6 lines · Chuck Lever
Quoted 5 lines
we still have a need to provide "iostat" like statistics for NFS clients. attached are a couple of patches, against 2.6.11.3, which prototype an approach for providing this kind of data to user programs.
i'd like some comment on the approach.
Quoted 6 lines
Quoted 3 lines
01-mountstats.patch adds a new file called
/proc/self/mountstatsand a new file system hook calledshow_stats. this just replicates/proc/mountsand theshow_optionshook.I don't think this makes a lot of sense. The stats aren't per namespace or mountpoint but per fs-instance, right?
Individual stats are per fs-instance, but the full collection _is_ per namespace.
Signature · 1 line
Trond Myklebust <[email protected]>