rhel5 rgmanager: Pause during exit if we stopped services

A difference between rgmanager 1.9.x and later versions is that they rely on openais/corosync for messaging. This messaging is quite reliable and has proved useful.

However, one drawback is that if you very quickly stop rgmanager and corosync/cman, the other nodes in the cluster can not restart services because message traffic is interrupted for the duration of the token timeout.

There is no simple solution to this problem. Rgmanager could (in theory) find new placements for services prior to stopping, but this is a large amount of design work; it was never designed to run policies in the exit path.

A far simpler idea is to simply give the other nodes time to restart services.

NOTE: This solution does not and can not work with central
processing mode.

Resolves: rhbz#619468

Signed-off-by Lon Hohberger <[email protected]>

---
 rgmanager/include/event.h        |    1 +
 rgmanager/include/resgroup.h     |    2 +-
 rgmanager/src/daemons/groups.c   |   15 +++++++++++++--
 rgmanager/src/daemons/main.c     |   14 ++++++++++++--
 rgmanager/src/daemons/rg_event.c |    7 +++++++
 5 files changed, 34 insertions(+), 5 deletions(-)

rgmanager/include/event.h +1 -0

index 7e628d8..e63dffd 100644
@@ -136,6 +136,7 @@ int slang_process_event(event_table_t *event_table, event_t *ev);
 
 /* For distributed events. */
 void set_transition_throttling(int nsecs);
+int get_transition_throttling(void);
 
 /* Simplified service start. */
 int service_op_start(char *svcName, int *target_list, int target_list_len,

rgmanager/include/resgroup.h +1 -1

index 793ad3b..4be4dbc 100644
@@ -160,7 +160,7 @@ void send_ret(msgctx_t *ctx, char *name, int ret, int req, int newowner);
 
 /* do this op on all resource groups.  The handler for the request 
    will sort out whether or not it's a valid request given the state */
-void rg_doall(int request, int block, char *debugfmt);
+int rg_doall(int request, int block, const char *debugfmt);
 void do_status_checks(void); /* Queue status checks for locally running
 				services */
 

rgmanager/src/daemons/groups.c +13 -2

index ecb7b85..b546421 100644
@@ -1292,12 +1292,21 @@ svc_exists(char *svcname)
 }
 
 
-void
-rg_doall(int request, int block, char *debugfmt)
+/*
+ * Perform an operation on all resources groups.
+ *
+ * Returns the number of requests queued.  This value is
+ * only used during shutdown, where we queue RG_STOP_EXITING
+ * only for services we have running locally as an optimization.
+ */
+int
+rg_doall(int request, int block,
+	 const char *debugfmt)
 {
 	resource_node_t *curr;
 	rg_state_t svcblk;
 	char rg[64];
+	int queued = 0;
 
 	pthread_rwlock_rdlock(&resource_lock);
 	list_do(&_tree, curr) {

@@ -1322,6 +1331,7 @@ rg_doall(int request, int block, char *debugfmt)
 
 		rt_enqueue_request(rg, request, NULL, 0,
 				   0, 0, 0);
+		++queued;
 	} while (!list_done(&_tree, curr));
 
 	pthread_rwlock_unlock(&resource_lock);

@@ -1331,6 +1341,7 @@ rg_doall(int request, int block, char *debugfmt)
 	   other rgmanagers to complete. */
 	if (block) 
 		rg_wait_threads();
+	return queued;
 }
 
 

rgmanager/src/daemons/main.c +12 -2

index aa78cef..1c7f746 100644
@@ -72,6 +72,7 @@ static int signalled = 0;
 static int port = RG_PORT;
 static char *rgmanager_lsname = "rgmanager"; /* XXX default */
 static int status_poll_interval = DEFAULT_CHECK_INTERVAL;
+static int stops_queued = 0;
 
 int next_node_id(cluster_member_list_t *membership, int me);
 

@@ -1041,7 +1042,7 @@ void *
 shutdown_thread(void __attribute__ ((unused)) *arg)
 {
 	rg_lockall(L_SYS|L_SHUTDOWN);
-	rg_doall(RG_STOP_EXITING, 1, NULL);
+	stops_queued = rg_doall(RG_STOP_EXITING, 1, NULL);
 	running = 0;
 
 	pthread_exit(NULL);

@@ -1219,8 +1220,17 @@ out_cleanup:
 		clu_lock_finished(rgmanager_lsname);
 
 out:
-	clulog(LOG_NOTICE, "Shutdown complete, exiting\n");
+	clulog(LOG_DEBUG, "Stopped %d services\n", stops_queued);
+	clulog(LOG_NOTICE, "Disconnecting from CMAN\n");
 	cman_finish(clu);
+
+	if (stops_queued && !central_events_enabled()) {
+		clulog(LOG_DEBUG, "Pausing to allow services to "
+		       "start on other node(s)\n");
+		sleep(get_transition_throttling() * 3);
+	}
+
+	clulog(LOG_NOTICE, "Exiting\n");
 
 	/*malloc_dump_table(); */ /* Only works if alloc.c us used */
 	/*malloc_stats();*/

rgmanager/src/daemons/rg_event.c +7 -0

index 82c20c0..606d41b 100644
@@ -69,6 +69,13 @@ set_transition_throttling(int nsecs)
 }
 
 
+int
+get_transition_throttling(void)
+{
+	return transition_throttling;
+}
+
+
 void
 set_central_events(int flag)
 {