dlm: Remove/bypass astd
While investigating Red Hat bug #537010 I started looking at the dlm's astd thread. The way in which the "cast" and "bast" requests are queued looked as if it might cause reordering since the "bast" requests are always delivered after any pending "cast" requests which is not always the correct ordering. This patch doesn't fix that bug, but it will prevent any races in that bit of code, and the performance benefits are also well worth having.
I noticed that astd seems to be extraneous to requirements. The notifications to astd are already running in process context, so they could be delivered directly. That should improve smp performance since all the notifications would no longer be funneled through a single thread.
Also, the only other function of astd seemed to be stopping the delivery of these notifications during recovery. Since, however, the notifications which are intercepted at recovery time are neither modified, nor filtered in any way, the only effect is to delay notifications for no obvious reason.
I thought that probably removing the astd thread and delivering the "cast" and "bast" notifications directly would improve performance due to the elimination of a scheduling delay. I wrote a small test module which creates a dlm lock space, and does 100,000 NL -> EX -> NL lock conversions.
Having run this test 10 times each on a 2.6.33-rc8 kernel and then the modified kernel including this patch, I got the following results:
Original: Avg time 24.62 us per conversion (NL -> EX -> NL)
Modified: Avg time 9.93 us per conversion
Which is a fairly dramatic speed up. Please consider applying this patch. I've tested it in both clustered and single node GFS2 configurations. The test figures are from a single node configuration which was a deliberate choice in order to avoid any effects of network latency.
--- fs/dlm/Makefile | 3 +- fs/dlm/ast.c | 165 ---------------------------------------------------- fs/dlm/ast.h | 26 -------- fs/dlm/lock.c | 16 ++++- fs/dlm/lockspace.c | 17 +----- fs/dlm/recover.c | 1 - fs/dlm/recoverd.c | 11 ---- 7 files changed, 15 insertions(+), 224 deletions(-) delete mode 100644 fs/dlm/ast.c delete mode 100644 fs/dlm/ast.h
fs/dlm/Makefile +1 -2
@@ -1,6 +1,5 @@ obj-$(CONFIG_DLM) += dlm.o -dlm-y := ast.o \ - config.o \ +dlm-y := config.o \ dir.o \ lock.o \ lockspace.o \
fs/dlm/ast.c deleted +0 -165
165 lines not on this page · raw message
fs/dlm/ast.h deleted +0 -26
@@ -1,26 +0,0 @@ -/****************************************************************************** -******************************************************************************* -** -** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. -** -** This copyrighted material is made available to anyone wishing to use, -** modify, copy, or redistribute it subject to the terms and conditions -** of the GNU General Public License v.2. -** -******************************************************************************* -******************************************************************************/ - -#ifndef __ASTD_DOT_H__ -#define __ASTD_DOT_H__ - -void dlm_add_ast (struct dlm_lkb *lkb,int type,int bastmode); -void dlm_del_ast (struct dlm_lkb *lkb); - -void dlm_astd_wake (void ); -int dlm_astd_start (void ); -void dlm_astd_stop (void ); -void dlm_astd_suspend (void ); -void dlm_astd_resume (void ); - -#endif -
fs/dlm/lock.c +13 -3
@@ -65,7 +65,6 @@#include "dir.h" #include "member.h" #include "lockspace.h" -#include "ast.h" #include "lock.h" #include "rcom.h" #include "recover.h"
@@ -283,6 +282,19 @@ static inline int is_overlap(struct dlm_lkb *lkb) DLM_IFL_OVERLAP_CANCEL)); } +static void dlm_add_ast (struct dlm_lkb *lkb,int type,int bastmode) +{ +if (lkb->lkb_flags &DLM_IFL_USER ) { +dlm_user_add_ast (lkb, type, bastmode); +return ; + } + +if (type &AST_COMP ) + lkb->lkb_astfn (lkb->lkb_astparam); +else if (type &AST_BAST ) + lkb->lkb_bastfn (lkb->lkb_astparam, bastmode); +} +static void queue_cast (struct dlm_rsb *r,struct dlm_lkb *lkb,int rv) {if (is_master_copy (lkb))
@@ -3854,8 +3866,6 @@ static void _receive_message(struct dlm_ls *ls, struct dlm_message *ms)default :log_error (ls,"unknown message type %d" , ms->m_type); } - -dlm_astd_wake (); }/* If the lockspace is in recovery mode (locking stopped), then normal
fs/dlm/lockspace.c +1 -16
@@ -15,7 +15,6 @@#include "lockspace.h" #include "member.h" #include "recoverd.h" -#include "ast.h" #include "dir.h" #include "lowcomms.h" #include "config.h"
@@ -356,16 +355,10 @@ static int threads_start(void)int error;/* Thread which process lock requests for all lockspace's */ - error =dlm_astd_start (); -if (error) { -log_print ("cannot start dlm_astd thread %d" , error); - goto fail; - } - error =dlm_scand_start ();if (error) {log_print ("cannot start dlm_scand thread %d" , error); - goto astd_fail; + goto fail; }/* Thread for sending/receiving messages for all lockspace's */
@@ -379,8 +372,6 @@ static int threads_start(void) scand_fail:dlm_scand_stop (); - astd_fail: -dlm_astd_stop (); fail:return error; }
@@ -389,7 +380,6 @@ static void threads_stop(void) {dlm_scand_stop ();dlm_lowcomms_stop (); -dlm_astd_stop (); } static int new_lockspace(const char *name, int namelen, void **lockspace,
@@ -708,8 +698,6 @@ static int release_lockspace(struct dlm_ls *ls, int force)dlm_delete_debug_file (ls); -dlm_astd_suspend (); -kfree (ls->ls_recover_buf);/*
@@ -731,15 +719,12 @@ static int release_lockspace(struct dlm_ls *ls, int force)list_del (&lkb->lkb_idtbl_list); -dlm_del_ast (lkb); -if (lkb->lkb_lvbptr && lkb->lkb_flags &DLM_IFL_MSTCPY )dlm_free_lvb (lkb->lkb_lvbptr);dlm_free_lkb (lkb); } } -dlm_astd_resume ();kfree (ls->ls_lkbtbl);
fs/dlm/recover.c +0 -1
@@ -15,7 +15,6 @@#include "lockspace.h" #include "dir.h" #include "config.h" -#include "ast.h" #include "memory.h" #include "rcom.h" #include "lock.h"
fs/dlm/recoverd.c +0 -11
@@ -15,7 +15,6 @@#include "lockspace.h" #include "member.h" #include "dir.h" -#include "ast.h" #include "recover.h" #include "lowcomms.h" #include "lock.h"
@@ -59,14 +58,6 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv)mutex_lock (&ls->ls_recoverd_active);/* -* Suspending and resuming dlm_astd ensures that no lkb's from this ls -* will be processed by dlm_astd during recovery. -*/ - -dlm_astd_suspend (); -dlm_astd_resume (); - -/* * Free non-master tossed rsb's. Master rsb's are kept on toss * list and put on root list to be included in resdir recovery. */
@@ -222,8 +213,6 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv)dlm_grant_after_purge (ls); -dlm_astd_wake (); -log_debug (ls,"recover %llx done: %u ms" , (unsigned long long )rv->seq,jiffies_to_msecs (jiffies - start));
Signature · 1 line
1.6.2.5
One of the reasons that ASTs are delivered in a separate thread was to allow ASTs do do other locking operations without causing a deadlock. eg. it would allow locks to be dropped or converted inside a blocking AST callback routine.
So maybe either the new code already allows for this or it's functionality that's not needed in the kernel. It should still be an option for userspace applications, but that's a different story altogether, of course
Chrissie
Quoted 37 lines · Steven Whitehouse
While investigating Red Hat bug #537010 I started looking at the dlm's astd thread. The way in which the "cast" and "bast" requests are queued looked as if it might cause reordering since the "bast" requests are always delivered after any pending "cast" requests which is not always the correct ordering. This patch doesn't fix that bug, but it will prevent any races in that bit of code, and the performance benefits are also well worth having.
I noticed that astd seems to be extraneous to requirements. The notifications to astd are already running in process context, so they could be delivered directly. That should improve smp performance since all the notifications would no longer be funneled through a single thread.
Also, the only other function of astd seemed to be stopping the delivery of these notifications during recovery. Since, however, the notifications which are intercepted at recovery time are neither modified, nor filtered in any way, the only effect is to delay notifications for no obvious reason.
I thought that probably removing the astd thread and delivering the "cast" and "bast" notifications directly would improve performance due to the elimination of a scheduling delay. I wrote a small test module which creates a dlm lock space, and does 100,000 NL -> EX -> NL lock conversions.
Having run this test 10 times each on a 2.6.33-rc8 kernel and then the modified kernel including this patch, I got the following results:
Original: Avg time 24.62 us per conversion (NL -> EX -> NL)
Modified: Avg time 9.93 us per conversionWhich is a fairly dramatic speed up. Please consider applying this patch. I've tested it in both clustered and single node GFS2 configurations. The test figures are from a single node configuration which was a deliberate choice in order to avoid any effects of network latency.
Signed-off-by Steven Whitehouse<[email protected]>---
Hi,
Quoted 5 lines · Christine Caulfield
One of the reasons that ASTs are delivered in a separate thread was to allow ASTs do do other locking operations without causing a deadlock. eg. it would allow locks to be dropped or converted inside a blocking AST callback routine.
Hmm... GFS2 doesn't require that at all, nor is it ever likely to since we have the glock layer to deal with that. I've looked at the OCFS2 code and I don't think they need it either - maybe Mark or Joel can confirm that for certain. Those are the only two users at the moment.
If it were to be the case that locking operations were being done in the context of the astd thread, the performance would be pretty poor since its a single thread no matter how many locks and lock spaces are in use. The only reasonable use for such a thing would also involve having to deal with the cache control for the locked object too (which for all current cases means disk I/O and/or cache invalidation), which would then also be limited to this single thread.
Quoted 7 lines
So maybe either the new code already allows for this or it's functionality that's not needed in the kernel. It should still be an option for userspace applications, but that's a different story altogether, of course
Chrissie
Yes, I've left the userspace interface code alone for now. That continues to work in the original way. My main concern is with the kernel interface at the moment,
Steve.
Quoted 38 lines · Steven Whitehouse
Quoted 37 lines
While investigating Red Hat bug #537010 I started looking at the dlm's astd thread. The way in which the "cast" and "bast" requests are queued looked as if it might cause reordering since the "bast" requests are always delivered after any pending "cast" requests which is not always the correct ordering. This patch doesn't fix that bug, but it will prevent any races in that bit of code, and the performance benefits are also well worth having.
I noticed that astd seems to be extraneous to requirements. The notifications to astd are already running in process context, so they could be delivered directly. That should improve smp performance since all the notifications would no longer be funneled through a single thread.
Also, the only other function of astd seemed to be stopping the delivery of these notifications during recovery. Since, however, the notifications which are intercepted at recovery time are neither modified, nor filtered in any way, the only effect is to delay notifications for no obvious reason.
I thought that probably removing the astd thread and delivering the "cast" and "bast" notifications directly would improve performance due to the elimination of a scheduling delay. I wrote a small test module which creates a dlm lock space, and does 100,000 NL -> EX -> NL lock conversions.
Having run this test 10 times each on a 2.6.33-rc8 kernel and then the modified kernel including this patch, I got the following results:
Original: Avg time 24.62 us per conversion (NL -> EX -> NL)
Modified: Avg time 9.93 us per conversionWhich is a fairly dramatic speed up. Please consider applying this patch. I've tested it in both clustered and single node GFS2 configurations. The test figures are from a single node configuration which was a deliberate choice in order to avoid any effects of network latency.
Signed-off-by Steven Whitehouse<[email protected]>---
David Teigland
maintainerQuoted 27 lines · Steven Whitehouse
While investigating Red Hat bug #537010 I started looking at the dlm's astd thread. The way in which the "cast" and "bast" requests are queued looked as if it might cause reordering since the "bast" requests are always delivered after any pending "cast" requests which is not always the correct ordering. This patch doesn't fix that bug, but it will prevent any races in that bit of code, and the performance benefits are also well worth having.
I noticed that astd seems to be extraneous to requirements. The notifications to astd are already running in process context, so they could be delivered directly. That should improve smp performance since all the notifications would no longer be funneled through a single thread.
Also, the only other function of astd seemed to be stopping the delivery of these notifications during recovery. Since, however, the notifications which are intercepted at recovery time are neither modified, nor filtered in any way, the only effect is to delay notifications for no obvious reason.
I thought that probably removing the astd thread and delivering the "cast" and "bast" notifications directly would improve performance due to the elimination of a scheduling delay. I wrote a small test module which creates a dlm lock space, and does 100,000 NL -> EX -> NL lock conversions.
Having run this test 10 times each on a 2.6.33-rc8 kernel and then the modified kernel including this patch, I got the following results:
Quoted 7 lines
Original: Avg time 24.62 us per conversion (NL -> EX -> NL)
Modified: Avg time 9.93 us per conversionWhich is a fairly dramatic speed up. Please consider applying this patch. I've tested it in both clustered and single node GFS2 configurations. The test figures are from a single node configuration which was a deliberate choice in order to avoid any effects of network latency.
Wow, there's no chance I'm going to even consider something like this. This would be a huge change in how the dlm has always operated, and would surely introduce very serious and hard to identify bugs (and ones that may not appear for a long time afterward). Given that there's *no problem* with the current method that has worked well for years, any change would be completely crazy.
Dave
Quoted 36 lines · David Teigland
Quoted 27 lines · Steven Whitehouse
While investigating Red Hat bug #537010 I started looking at the dlm's astd thread. The way in which the "cast" and "bast" requests are queued looked as if it might cause reordering since the "bast" requests are always delivered after any pending "cast" requests which is not always the correct ordering. This patch doesn't fix that bug, but it will prevent any races in that bit of code, and the performance benefits are also well worth having.
I noticed that astd seems to be extraneous to requirements. The notifications to astd are already running in process context, so they could be delivered directly. That should improve smp performance since all the notifications would no longer be funneled through a single thread.
Also, the only other function of astd seemed to be stopping the delivery of these notifications during recovery. Since, however, the notifications which are intercepted at recovery time are neither modified, nor filtered in any way, the only effect is to delay notifications for no obvious reason.
I thought that probably removing the astd thread and delivering the "cast" and "bast" notifications directly would improve performance due to the elimination of a scheduling delay. I wrote a small test module which creates a dlm lock space, and does 100,000 NL -> EX -> NL lock conversions.
Having run this test 10 times each on a 2.6.33-rc8 kernel and then the modified kernel including this patch, I got the following results:
Quoted 7 lines
Original: Avg time 24.62 us per conversion (NL -> EX -> NL)
Modified: Avg time 9.93 us per conversionWhich is a fairly dramatic speed up. Please consider applying this patch. I've tested it in both clustered and single node GFS2 configurations. The test figures are from a single node configuration which was a deliberate choice in order to avoid any effects of network latency.
Quoted 6 lines
Wow, there's no chance I'm going to even consider something like this. This would be a huge change in how the dlm has always operated, and would surely introduce very serious and hard to identify bugs (and ones that may not appear for a long time afterward). Given that there's *no problem* with the current method that has worked well for years, any change would be completely crazy.
I think total dismissal of the idea is unreasonable. At least give it a chance, and test it for a while. Your argument is valid for not including it in a stable release as yet, but not for ignoring it completely.
Here we have a chance to remove some code, a kernel thread AND improve performance hugely. Just saying "it's always worked like that and we're not changing it" is the crazy argument IMHO.
Chrissie