Skip to contentSkip to conversation

Fix: join consumer timer thread

PATCH New Mathieu Desnoyers 2 participants 2 messages Jun 16, 2017 – Jun 21, 2017 1 message not in the archive

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH 2/2 Fix: join consumer timer thread

Detaching the timer thread has the unfortunate side-effect of letting the health management data structures be freed by main() while the timer thread may still be using them (if, e.g., main() exits quickly).

Overcome this situation by tearing down and joining the timer thread.

---
 src/bin/lttng-consumerd/lttng-consumerd.c | 68 +++++++++++++++++++------------
 src/common/consumer/consumer-timer.c      | 13 ++++--
 src/common/consumer/consumer-timer.h      |  1 +
 3 files changed, 52 insertions(+), 30 deletions(-)
src/bin/lttng-consumerd/lttng-consumerd.c +42 -26
index 9fb4747..2a8b092 100644
@@ -56,6 +56,7 @@
 
 static pthread_t channel_thread, data_thread, metadata_thread,
 		sessiond_thread, metadata_timer_thread, health_thread;
+static bool metadata_timer_thread_online;
 
 /* to count the number of times the user pressed ctrl+c */
 static int sigintcount = 0;
@@ -506,6 +507,20 @@ int main(int argc, char **argv)
 	}
 	cmm_smp_mb();	/* Read ready before following operations */
 
+	/*
+	 * Create the thread to manage the UST metadata periodic timer and
+	 * live timer.
+	 */
+	ret = pthread_create(&metadata_timer_thread, default_pthread_attr(),
+			consumer_timer_thread, (void *) ctx);
+	if (ret) {
+		errno = ret;
+		PERROR("pthread_create");
+		retval = -1;
+		goto exit_metadata_timer_thread;
+	}
+	metadata_timer_thread_online = true;
+
 	/* Create thread to manage channels */
 	ret = pthread_create(&channel_thread, default_pthread_attr(),
 			consumer_thread_channel_poll,
@@ -549,34 +564,12 @@ int main(int argc, char **argv)
 		goto exit_sessiond_thread;
 	}
 
-	/*
-	 * Create the thread to manage the UST metadata periodic timer and
-	 * live timer.
-	 */
-	ret = pthread_create(&metadata_timer_thread, default_pthread_attr(),
-			consumer_timer_thread, (void *) ctx);
-	if (ret) {
-		errno = ret;
-		PERROR("pthread_create");
-		retval = -1;
-		goto exit_metadata_timer_thread;
-	}
-
-	ret = pthread_detach(metadata_timer_thread);
-	if (ret) {
-		errno = ret;
-		PERROR("pthread_detach");
-		retval = -1;
-		goto exit_metadata_timer_detach;
-	}
 
 	/*
 	 * This is where we start awaiting program completion (e.g. through
 	 * signal that asks threads to teardown.
 	 */
 
-exit_metadata_timer_detach:
-exit_metadata_timer_thread:
 	ret = pthread_join(sessiond_thread, &status);
 	if (ret) {
 		errno = ret;
@@ -617,6 +610,8 @@ exit_metadata_thread:
 	}
 exit_channel_thread:
 
+exit_metadata_timer_thread:
+
 	ret = pthread_join(health_thread, &status);
 	if (ret) {
 		errno = ret;
@@ -629,17 +624,38 @@ exit_health_thread:
 exit_health_pipe:
 
 exit_init_data:
-	tmp_ctx = ctx;
-	ctx = NULL;
-	cmm_barrier();	/* Clear ctx for signal handler. */
 	/*
 	 * Wait for all pending call_rcu work to complete before tearing
 	 * down data structures. call_rcu worker may be trying to
 	 * perform lookups in those structures.
 	 */
 	rcu_barrier();
-	lttng_consumer_destroy(tmp_ctx);
 	lttng_consumer_cleanup();
+	/*
+	 * Tearing down the metadata timer thread in a
+	 * non-fully-symmetric fashion compared to its creation in case
+	 * lttng_consumer_cleanup() ends up tearing down timers (which
+	 * requires the timer thread to be alive).
+	 */
+	if (metadata_timer_thread_online) {
+		/*
+		 * Ensure the metadata timer thread exits only after all other
+		 * threads are gone, because it is required to perform timer
+		 * teardown synchronization.
+		 */
+		kill(getpid(), LTTNG_CONSUMER_SIG_EXIT);
+		ret = pthread_join(metadata_timer_thread, &status);
+		if (ret) {
+			errno = ret;
+			PERROR("pthread_join metadata_timer_thread");
+			retval = -1;
+		}
+		metadata_timer_thread_online = false;
+	}
+	tmp_ctx = ctx;
+	ctx = NULL;
+	cmm_barrier();	/* Clear ctx for signal handler. */
+	lttng_consumer_destroy(tmp_ctx);
 
 	if (health_consumerd) {
 		health_app_destroy(health_consumerd);
src/common/consumer/consumer-timer.c +9 -4
index 0adc572..e42940e 100644
@@ -72,6 +72,10 @@ static void setmask(sigset_t *mask)
 	if (ret) {
 		PERROR("sigaddset monitor");
 	}
+	ret = sigaddset(mask, LTTNG_CONSUMER_SIG_EXIT);
+	if (ret) {
+		PERROR("sigaddset exit");
+	}
 }
 
 static int channel_monitor_pipe = -1;
@@ -782,7 +786,7 @@ end:
 /*
  * This thread is the sighandler for signals LTTNG_CONSUMER_SIG_SWITCH,
  * LTTNG_CONSUMER_SIG_TEARDOWN, LTTNG_CONSUMER_SIG_LIVE, and
- * LTTNG_CONSUMER_SIG_MONITOR.
+ * LTTNG_CONSUMER_SIG_MONITOR, LTTNG_CONSUMER_SIG_EXIT.
  */
 void *consumer_timer_thread(void *data)
 {
@@ -836,6 +840,9 @@ void *consumer_timer_thread(void *data)
 
 			channel = info.si_value.sival_ptr;
 			monitor_timer(ctx, channel);
+		} else if (signr == LTTNG_CONSUMER_SIG_EXIT) {
+			assert(CMM_LOAD_SHARED(consumer_quit));
+			goto end;
 		} else {
 			ERR("Unexpected signal %d\n", info.si_signo);
 		}
@@ -844,10 +851,8 @@ void *consumer_timer_thread(void *data)
 error_testpoint:
 	/* Only reached in testpoint error */
 	health_error();
+end:
 	health_unregister(health_consumerd);
-
 	rcu_unregister_thread();
-
-	/* Never return */
 	return NULL;
 }
src/common/consumer/consumer-timer.h +1 -0
index 851a172..1b5dd41 100644
@@ -28,6 +28,7 @@
 #define LTTNG_CONSUMER_SIG_TEARDOWN	SIGRTMIN + 11
 #define LTTNG_CONSUMER_SIG_LIVE		SIGRTMIN + 12
 #define LTTNG_CONSUMER_SIG_MONITOR	SIGRTMIN + 13
+#define LTTNG_CONSUMER_SIG_EXIT		SIGRTMIN + 14
 
 #define CLOCKID CLOCK_MONOTONIC
 
Signature · 6 lines
2.1.4

_______________________________________________
lttng-dev mailing list
[email protected]
https://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
#2 lore raw

Re: [PATCH lttng-tools 1/2] Cleanup: use CMM accessors for consumer_quit variable

In reply to a message not in the archive

The first patch has been merged in master and stable-2.10. The second one has been merged in master, stable-2.10, stable-2.9, and stable-2.8.

Thanks!
Jérémie

Quoted 108 lines · Mathieu Desnoyers

Use CMM_LOAD_SHARED and CMM_STORE_SHARED, which are strictly equivalent to a volatile variable, in line with the rest of the lttng-tools project.

Also move its declaration to a header, rather than having multiple declarations in C files, now following our coding style.

---
 src/common/consumer/consumer.c               | 10 +++++-----
 src/common/consumer/consumer.h               |  6 ++++++
 src/common/kernel-consumer/kernel-consumer.c |  1 -
 src/common/ust-consumer/ust-consumer.c       |  1 -
 4 files changed, 11 insertions(+), 7 deletions(-)
src/common/consumer/consumer.c +5 -5
index d07e5b4..9834322 100644
@@ -76,7 +76,7 @@ int data_consumption_paused;
  * Also updated by the signal handler (consumer_should_exit()). Read by the
  * polling threads.
  */
-volatile int consumer_quit;
+int consumer_quit;
 
 /*
  * Global hash table containing respectively metadata and data streams. The
@@ -1227,7 +1227,7 @@ void lttng_consumer_should_exit(struct lttng_consumer_local_data *ctx)
 {
        ssize_t ret;
 
-       consumer_quit = 1;
+       CMM_STORE_SHARED(consumer_quit, 1);
        ret = lttng_write(ctx->consumer_should_quit[1], "4", 1);
        if (ret < 1) {
                PERROR("write consumer quit");
@@ -2526,7 +2526,7 @@ void *consumer_thread_data_poll(void *data)
                pthread_mutex_unlock(&consumer_data.lock);
 
                /* No FDs and consumer_quit, consumer_cleanup the thread */
-               if (nb_fd == 0 && consumer_quit == 1) {
+               if (nb_fd == 0 && CMM_LOAD_SHARED(consumer_quit) == 1) {
                        err = 0;        /* All is OK */
                        goto end;
                }
@@ -3203,7 +3203,7 @@ void *consumer_thread_sessiond_poll(void *data)
                        err = 0;
                        goto end;
                }
-               if (consumer_quit) {
+               if (CMM_LOAD_SHARED(consumer_quit)) {
                        DBG("consumer_thread_receive_fds received quit from signal");
                        err = 0;        /* All is OK */
                        goto end;
@@ -3228,7 +3228,7 @@ end:
         * when all fds have hung up, the polling thread
         * can exit cleanly
         */
-       consumer_quit = 1;
+       CMM_STORE_SHARED(consumer_quit, 1);
 
        /*
         * Notify the data poll thread to poll back again and test the
src/common/consumer/consumer.h +6 -0
index 7375373..220dbb4 100644
@@ -604,6 +604,12 @@ struct lttng_consumer_global_data {
        struct lttng_ht *stream_per_chan_id_ht;
 };
 
+/*
+ * Set to nonzero when the consumer is exiting. Updated by signal
+ * handler and thread exit, read by threads.
+ */
+extern int consumer_quit;
+
 /* Flag used to temporarily pause data consumption from testpoints. */
 extern int data_consumption_paused;
 
src/common/kernel-consumer/kernel-consumer.c +0 -1
index b5afc73..8d00a0d 100644
@@ -48,7 +48,6 @@
 
 extern struct lttng_consumer_global_data consumer_data;
 extern int consumer_poll_timeout;
-extern volatile int consumer_quit;
 
 /*
  * Take a snapshot for a specific fd
src/common/ust-consumer/ust-consumer.c +0 -2
index 99803a6..4297d60 100644
@@ -51,7 +51,6 @@
 
 extern struct lttng_consumer_global_data consumer_data;
 extern int consumer_poll_timeout;
-extern volatile int consumer_quit;
 
 /*
  * Free channel object and all streams associated with it. This MUST be used
--

2.1.4

Signature · 7 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com
_______________________________________________
lttng-dev mailing list
[email protected]
https://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev