Remove BKL from NFS read/write code + SunRPC...
Subsystems (3)
PATCH 1/1 Remove BKL from NFS read/write code + SunRPC...
The following patch strongly reduces BKL contention within the NFS
read/write code, and within the generic RPC layer. Features:
- BKL is added explicitly where needed for list protection in the NLM locking code.
- BKL added where still needed to protect the NFS_FLAGS(inode) (this will of course be fixed in a later patch).
- Replace BKL with spinlock "nfs_flushd_lock" for protecting the read/write asynchronous flush daemon.
- Replace BKL with spinlock "nfs_inode_lock" for protecting the inode attribute updates.
This also allows us to drop the BKL from the SunRPC code. For the latter, all the internals should already be SMP safe. The global BKL was only being maintained in order to ensure that the user callbacks don't race with one another (and because I was nervous of removing it in a stable series ;-).
Cheers,
Trond
fs/lockd/clntproc.c +8 -0
@@ -569,11 +569,15 @@printk (KERN_WARNING "lockd: unexpected unlock status: %d\n" , status); die: +lock_kernel ();nlm_release_host (req->a_host); +unlock_kernel ();kfree (req);return ; retry_rebind: +lock_kernel ();nlm_rebind_host (req->a_host); +unlock_kernel (); retry_unlock:rpc_restart_call (task); }
@@ -650,12 +654,16 @@ } die: +lock_kernel ();nlm_release_host (req->a_host); +unlock_kernel ();kfree (req);return ; retry_cancel: +lock_kernel ();nlm_rebind_host (req->a_host); +unlock_kernel ();rpc_restart_call (task);rpc_delay (task,30 *HZ ); }
fs/lockd/svc4proc.c +3 -0
@@ -17,6 +17,7 @@#include <linux/lockd/lockd.h> #include <linux/lockd/share.h> #include <linux/lockd/sm_inter.h> +#include <linux/smp_lock.h> #define NLMDBG_FACILITY NLMDBG_CLIENT
@@ -499,7 +500,9 @@dprintk ("lockd: %4d callback failed (errno = %d)\n" , task->tk_pid, -task->tk_status); } +lock_kernel ();nlm_release_host (call->a_host); +unlock_kernel ();kfree (call); }
fs/lockd/svclock.c +4 -1
@@ -576,9 +576,10 @@dprintk ("lockd: GRANT_MSG RPC callback\n" );dprintk ("callback: looking for cookie %x \n" , *(unsigned int *)(call->a_args.cookie.data)); +lock_kernel ();if (!(block =nlmsvc_find_block (&call->a_args.cookie))) {dprintk ("lockd: no block for cookie %x\n" , *(u32 *)(call->a_args.cookie.data)); -return ; + goto out; }/* Technically, we should down the file semaphore here. Since we
@@ -599,6 +600,8 @@ block->b_incall =0 ;nlm_release_host (call->a_host); + out: +unlock_kernel (); }/*
fs/lockd/svcproc.c +3 -0
@@ -18,6 +18,7 @@#include <linux/lockd/lockd.h> #include <linux/lockd/share.h> #include <linux/lockd/sm_inter.h> +#include <linux/smp_lock.h> #define NLMDBG_FACILITY NLMDBG_CLIENT
@@ -527,7 +528,9 @@dprintk ("lockd: %4d callback failed (errno = %d)\n" , task->tk_pid, -task->tk_status); } +lock_kernel ();nlm_release_host (call->a_host); +unlock_kernel ();kfree (call); }
fs/nfs/file.c +7 -8
@@ -99,7 +99,9 @@ dentry->d_parent->d_name.name, dentry->d_name.name, (unsigned long) count, (unsigned long) *ppos); +lock_kernel (); result =nfs_revalidate_inode (NFS_SERVER (inode), inode); +unlock_kernel ();if (!result) result =generic_file_read (file, buf, count, ppos);return result;
@@ -115,7 +117,9 @@dfprintk (VFS ,"nfs: mmap(%s/%s)\n" , dentry->d_parent->d_name.name, dentry->d_name.name); +lock_kernel (); status =nfs_revalidate_inode (NFS_SERVER (inode), inode); +unlock_kernel ();if (!status) status =generic_file_mmap (file, vma);return status;
@@ -134,13 +138,11 @@dfprintk (VFS ,"nfs: fsync(%s/%ld)\n" , inode->i_sb->s_id, inode->i_ino); -lock_kernel (); status =nfs_wb_file (inode, file);if (!status) { status = file->f_error; file->f_error =0 ; } -unlock_kernel ();return status; }
@@ -160,12 +162,7 @@static int nfs_commit_write (struct file *file,struct page *page,unsigned offset,unsigned to) { -long status; - -lock_kernel (); - status =nfs_updatepage (file, page, offset, to-offset); -unlock_kernel (); -return status; +return nfs_updatepage (file, page, offset, to-offset); }/*
@@ -219,7 +216,9 @@ result = -EBUSY ;if (IS_SWAPFILE (inode)) goto out_swapfile; +lock_kernel (); result =nfs_revalidate_inode (NFS_SERVER (inode), inode); +unlock_kernel ();if (result) goto out;
fs/nfs/flushd.c +28 -10
@@ -51,6 +51,19 @@* This is the wait queue all cluster daemons sleep on */ static struct rpc_wait_queue flushd_queue =RPC_INIT_WAITQ ("nfs_flushd" ); +static spinlock_t nfs_flushd_lock =SPIN_LOCK_UNLOCKED ; + +static inline void +nfs_lock_flushd (void ) +{ +spin_lock (&nfs_flushd_lock); +} + +static inline void +nfs_unlock_flushd (void ) +{ +spin_unlock (&nfs_flushd_lock); +}/* * Local function declarations.
@@ -67,12 +80,11 @@dprintk ("NFS: writecache_init\n" ); -lock_kernel (); - status = -ENOMEM ;/* Create the RPC task */ if (!(task =rpc_new_task (server->client,NULL ,RPC_TASK_ASYNC ))) - goto out_unlock; +return -ENOMEM ; +nfs_lock_flushd (); cache = server->rw_requests; status =0 ;
@@ -89,22 +101,21 @@ cache->auth = server->client->cl_auth; task->tk_action = nfs_flushd; task->tk_exit = nfs_flushd_exit; +nfs_unlock_flushd ();rpc_execute (task); -unlock_kernel ();return 0 ; out_unlock: -if (task) -rpc_release_task (task); -unlock_kernel (); -return status; +nfs_unlock_flushd (); +rpc_release_task (task); +return 0 ; }void nfs_reqlist_exit (struct nfs_server *server) {struct nfs_reqlist *cache; -lock_kernel (); +nfs_lock_flushd (); cache = server->rw_requests;if (!cache) goto out;
@@ -114,11 +125,13 @@while (cache->task) {rpc_exit (cache->task,0 );rpc_wake_up_task (cache->task); +nfs_unlock_flushd ();interruptible_sleep_on_timeout (&cache->request_wait,1 *HZ ); +nfs_lock_flushd (); } out: -unlock_kernel (); +nfs_unlock_flushd (); }int nfs_reqlist_alloc (struct nfs_server *server)
@@ -183,11 +196,13 @@ }dprintk ("NFS: %4d flushd back to sleep\n" , task->tk_pid); +nfs_lock_flushd ();if (task->tk_action) { task->tk_timeout =NFS_FLUSHD_TIMEOUT ; cache->runat = jiffies + task->tk_timeout;rpc_sleep_on (&flushd_queue, task,NULL ,NULL ); } +nfs_unlock_flushd (); } static void
@@ -196,10 +211,13 @@struct nfs_server *server;struct nfs_reqlist *cache; server = (struct nfs_server *) task->tk_calldata; + +nfs_lock_flushd (); cache = server->rw_requests;if (cache->task == task) cache->task =NULL ;wake_up (&cache->request_wait); +nfs_unlock_flushd (); }
fs/nfs/inode.c +40 -11
130 lines not on this page · raw message
fs/nfs/read.c +0 -4
@@ -114,11 +114,9 @@ (long long )NFS_FILEID (inode), (long long)offset, rsize, buffer); -lock_kernel (); result =NFS_PROTO (inode)->read (inode, cred, &fattr, flags, offset, rsize, buffer, &eof);nfs_refresh_inode (inode, &fattr); -unlock_kernel ();/* * Even if we had a partial success we can't mark the page
@@ -277,9 +275,7 @@rpc_clnt_sigmask (clnt, &oldset);rpc_call_setup (task, &msg,0 ); -lock_kernel ();rpc_execute (task); -unlock_kernel ();rpc_clnt_sigunmask (clnt, &oldset);return 0 ; out_bad:
fs/nfs/write.c +3 -13
@@ -194,8 +194,7 @@* If we've extended the file, update the inode * now so we don't invalidate the cache. */ -if (base > inode->i_size) - inode->i_size = base; +nfs_grow_isize (inode, base); }while (count);if (PageError (page))
@@ -226,9 +225,7 @@nfs_unlock_request (req);nfs_strategy (inode); end = ((loff_t )page->index<<PAGE_CACHE_SHIFT ) + (loff_t)(offset + count); -if (inode->i_size < end) - inode->i_size = end; - +nfs_grow_isize (inode, end); out:return status; }
@@ -260,7 +257,6 @@if (page->index >= end_index+1 || !offset) goto out; do_it: -lock_kernel ();if (NFS_SERVER (inode)->wsize >=PAGE_CACHE_SIZE && !IS_SYNC (inode)) { err =nfs_writepage_async (NULL , inode, page,0 , offset);if (err >=0 )
@@ -270,7 +266,6 @@if (err == offset) err =0 ; } -unlock_kernel (); out:UnlockPage (page);return err;
@@ -828,8 +823,7 @@ status =0 ; end = ((loff_t )page->index<<PAGE_CACHE_SHIFT ) + (loff_t)(offset + count); -if (inode->i_size < end) - inode->i_size = end; +nfs_grow_isize (inode, end);/* If we wrote past the end of the page. * Call the strategy routine so it can send out a bunch
@@ -952,9 +946,7 @@rpc_clnt_sigmask (clnt, &oldset);rpc_call_setup (task, &msg,0 ); -lock_kernel ();rpc_execute (task); -unlock_kernel ();rpc_clnt_sigunmask (clnt, &oldset);return 0 ; out_bad:
@@ -1175,9 +1167,7 @@dprintk ("NFS: %4d initiated commit call\n" , task->tk_pid);rpc_clnt_sigmask (clnt, &oldset);rpc_call_setup (task, &msg,0 ); -lock_kernel ();rpc_execute (task); -unlock_kernel ();rpc_clnt_sigunmask (clnt, &oldset);return 0 ; out_bad:
include/linux/nfs_fs.h +3 -4
@@ -188,10 +188,7 @@#define NFS_CACHE_MTIME (inode) (NFS_I(inode)->read_cache_mtime)#define NFS_CACHE_ISIZE (inode) (NFS_I(inode)->read_cache_isize)#define NFS_NEXTSCAN (inode) (NFS_I(inode)->nextscan) -#define NFS_CACHEINV (inode) \ -do { \ - NFS_READTIME(inode) = jiffies - NFS_MAXATTRTIMEO(inode) - 1; \ -} while (0) +#define NFS_CACHEINV (inode) nfs_invalidate_caches(inode)#define NFS_ATTRTIMEO (inode) (NFS_I(inode)->attrtimeo)#define NFS_MINATTRTIMEO (inode) \ (S_ISDIR(inode->i_mode)? NFS_SERVER(inode)->acdirmin \
@@ -227,11 +224,13 @@* linux/fs/nfs/inode.c */ extern void nfs_zap_caches (struct inode *); +extern void nfs_invalidate_caches (struct inode *);extern int nfs_inode_is_stale (struct inode *,struct nfs_fh *,struct nfs_fattr *);extern struct inode *nfs_fhget (struct dentry *,struct nfs_fh *,struct nfs_fattr *);extern int __nfs_refresh_inode (struct inode *,struct nfs_fattr *); +extern void nfs_grow_isize (struct inode *,loff_t );extern int nfs_revalidate (struct dentry *);extern int nfs_permission (struct inode *,int );extern int nfs_open (struct inode *,struct file *);
net/sunrpc/sched.c +0 -1
@@ -1052,7 +1052,6 @@int rounds =0 ;MOD_INC_USE_COUNT ; -lock_kernel ();/* * Let our maker know we're running ... */
Quoted 2 lines · Trond Myklebust
The following patch strongly reduces BKL contention within the NFS read/write code, and within the generic RPC layer.
Do you have any benchmarks which showed BKL contention in the NFS code?
I'm not trying to criticize, I think the patch is wonderful. I want
to have some more numbers to say, "Look! The BKL _is_ bad!"
Signature · 2 lines
Dave Hansen [email protected]
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
" " == Dave Hansen <[email protected]> writes:
> Trond Myklebust wrote:
>> The following patch strongly reduces BKL contention within the >> NFS read/write code, and within the generic RPC layer.
> Do you have any benchmarks which showed BKL contention in the > NFS code?
> I'm not trying to criticize, I think the patch is wonderful.
> I want
> to have some more numbers to say, "Look! The BKL _is_ bad!"
See Chuck's paper on
http://www.citi.umich.edu/projects/nfs-perf/results/cel/write-throughput.html
Cheers,
Trond
there's also a technical report that describes work we did recently to improve write performance, which summarizes the web page Trond listed below. see
http://www.citi.umich.edu/techreports/reports/citi-tr-01-12.pdf
we're publishing a revised version in the June 2002 Usenix technical conference proceedings; if you intend to cite the paper, we'd prefer that you cited that one instead.
Quoted 12 lines · Trond Myklebust
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
" " == Dave Hansen <[email protected]> writes:
> Trond Myklebust wrote:
>> The following patch strongly reduces BKL contention within the >> NFS read/write code, and within the generic RPC layer.> Do you have any benchmarks which showed BKL contention in the > NFS code?
> I'm not trying to criticize, I think the patch is wonderful.
> I want
> to have some more numbers to say, "Look! The BKL _is_ bad!"
Quoted 8 lines
See Chuck's paper on
http://www.citi.umich.edu/projects/nfs-perf/results/cel/write-throughput.html
Cheers,
Trond
- Chuck Lever
--
corporate: <[email protected]>
personal: <[email protected]>