Skip to contentSkip to conversation

Fix: consumerd: consumer_stream_write_index error handling

PATCH New Mathieu Desnoyers 2 participants 9 messages Sep 9, 2015 – Sep 14, 2015 5 messages not in the archive

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH 2/5 Fix: relayd: handle consumerd crashes without leak

We can be clever about indexes partially received in cases where we received the data socket part, but not the control socket part: since we're currently closing the stream on behalf of the control socket, we *know* there won't be any more control information for this socket. Therefore, we can destroy all indexes for which we have received only the file descriptor (from data socket). This takes care of consumerd crashes between sending the data and control information for a packet. Since those are sent in that order, we take care of consumerd crashes.

---
 src/bin/lttng-relayd/index.c  | 39 +++++++++++++++++++++++++++
 src/bin/lttng-relayd/index.h  |  2 ++
 src/bin/lttng-relayd/stream.c | 61 +++++++++++++++++++++++++++++++++----------
 3 files changed, 88 insertions(+), 14 deletions(-)
src/bin/lttng-relayd/index.c +39 -0
index 7182e36..cb7ae3d 100644
@@ -333,3 +333,42 @@ void relay_index_close_all(struct relay_stream *stream)
 	}
 	rcu_read_unlock();
 }
+
+void relay_index_close_partial_fd(struct relay_stream *stream)
+{
+	struct lttng_ht_iter iter;
+	struct relay_index *index;
+
+	rcu_read_lock();
+	cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter,
+			index, index_n.node) {
+		if (!index->index_fd) {
+			continue;
+		}
+		/*
+		 * Partial index has its index_fd: we have only
+		 * received its info from the data socket.
+		 * Put self-ref from index.
+		 */
+		relay_index_put(index);
+	}
+	rcu_read_unlock();
+}
+
+uint64_t relay_index_find_last(struct relay_stream *stream)
+{
+	struct lttng_ht_iter iter;
+	struct relay_index *index;
+	uint64_t net_seq_num = -1ULL;
+
+	rcu_read_lock();
+	cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter,
+			index, index_n.node) {
+		if (net_seq_num == -1ULL ||
+				index->index_n.key > net_seq_num) {
+			net_seq_num = index->index_n.key;
+		}
+	}
+	rcu_read_unlock();
+	return net_seq_num;
+}
src/bin/lttng-relayd/index.h +2 -0
index e882ed9..15c4ac8 100644
@@ -71,5 +71,7 @@ int relay_index_set_data(struct relay_index *index,
 int relay_index_try_flush(struct relay_index *index);
 
 void relay_index_close_all(struct relay_stream *stream);
+void relay_index_close_partial_fd(struct relay_stream *stream);
+uint64_t relay_index_find_last(struct relay_stream *stream);
 
 #endif /* _RELAY_INDEX_H */
src/bin/lttng-relayd/stream.c +47 -14

73 lines not on this page · raw message

Signature · 1 line
2.1.4
#2 lore raw

PATCH 3/5 Fix: consumerd: consumer_stream_write_index error handling

In reply to a message not in the archive

When consumer_stream_write_index cannot find the relay daemon, it currently assumes that the reason for this situation is that it should write to a file. Add a check to see if indeed we should write to a file, of if this is caused by a removed relayd. Return an error and print an error to the console in this situation.

---
 src/common/consumer-stream.c | 8 ++++++++
 1 file changed, 8 insertions(+)
src/common/consumer-stream.c +8 -0
index 604e2f4..23cc882 100644
@@ -374,10 +374,18 @@ int consumer_stream_write_index(struct lttng_consumer_stream *stream,
 		pthread_mutex_lock(&relayd->ctrl_sock_mutex);
 		ret = relayd_send_index(&relayd->control_sock, index,
 				stream->relayd_stream_id, stream->next_net_seq_num - 1);
+
 		pthread_mutex_unlock(&relayd->ctrl_sock_mutex);
 	} else {
 		ssize_t size_ret;
 
+		if (stream->net_seq_idx != -1ULL) {
+			ERR("Cannot find relayd for write index, net_seq_idx %" PRIu64,
+				stream->net_seq_idx);
+			ret = -1;
+			goto error;
+		}
+
 		size_ret = index_write(stream->index_fd, index,
 				sizeof(struct ctf_packet_index));
 		if (size_ret < sizeof(struct ctf_packet_index)) {
Signature · 1 line
2.1.4
#3 lore raw

PATCH 4/5 Enhance relayd error reporting

In reply to a message not in the archive

relay_process_data has error cases that don't print any error to the console. Add those cases, and enhance the information provided by error output within handle_index_data().

---
 src/bin/lttng-relayd/main.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
src/bin/lttng-relayd/main.c +9 -2
index dcaaaa8..0c7ef8d 100644
@@ -2084,8 +2084,8 @@ static int handle_index_data(struct relay_stream *stream, uint64_t net_seq_num,
 	/* Get data offset because we are about to update the index. */
 	data_offset = htobe64(stream->tracefile_size_current);
 
-	DBG("handle_index_data: stream %" PRIu64 " data offset %" PRIu64,
-		stream->stream_handle, stream->tracefile_size_current);
+	DBG("handle_index_data: stream %" PRIu64 " net_seq_num %" PRIu64 " data offset %" PRIu64,
+			stream->stream_handle, net_seq_num, stream->tracefile_size_current);
 
 	/*
 	 * Lookup for an existing index for that stream id/sequence
@@ -2183,6 +2183,7 @@ static int relay_process_data(struct relay_connection *conn)
 	stream_id = be64toh(data_hdr.stream_id);
 	stream = stream_get_by_id(stream_id);
 	if (!stream) {
+		ERR("relay_process_data: Cannot find stream %" PRIu64, stream_id);
 		ret = -1;
 		goto end;
 	}
@@ -2212,6 +2213,8 @@ static int relay_process_data(struct relay_connection *conn)
 		if (ret == 0) {
 			/* Orderly shutdown. Not necessary to print an error. */
 			DBG("Socket %d did an orderly shutdown", conn->sock->fd);
+		} else {
+			ERR("Socket %d error %d", conn->sock->fd, ret);
 		}
 		ret = -1;
 		goto end_stream_put;
@@ -2255,6 +2258,8 @@ static int relay_process_data(struct relay_connection *conn)
 	if (session->minor >= 4 && !session->snapshot) {
 		ret = handle_index_data(stream, net_seq_num, rotate_index);
 		if (ret < 0) {
+			ERR("handle_index_data: fail stream %" PRIu64 " net_seq_num %" PRIu64 " ret %d",
+					stream->stream_handle, net_seq_num, ret);
 			goto end_stream_unlock;
 		}
 	}
@@ -2273,6 +2278,8 @@ static int relay_process_data(struct relay_connection *conn)
 	ret = write_padding_to_file(stream->stream_fd->fd,
 			be32toh(data_hdr.padding_size));
 	if (ret < 0) {
+		ERR("write_padding_to_file: fail stream %" PRIu64 " net_seq_num %" PRIu64 " ret %d",
+				stream->stream_handle, net_seq_num, ret);
 		goto end_stream_unlock;
 	}
 	stream->tracefile_size_current +=
Signature · 1 line
2.1.4
#4 lore raw

PATCH 5/5 Print relayd stream indexes

In reply to a message not in the archive

In debug output, print leaking stream index information when exiting from relay daemon.

---
 src/bin/lttng-relayd/stream.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
src/bin/lttng-relayd/stream.c +22 -0
index cac8763..b604919 100644
@@ -416,6 +416,27 @@ void try_stream_close(struct relay_stream *stream)
 	stream_put(stream);
 }
 
+static void print_stream_indexes(struct relay_stream *stream)
+{
+	struct lttng_ht_iter iter;
+	struct relay_index *index;
+
+	rcu_read_lock();
+	cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter, index,
+			index_n.node) {
+		DBG("index %p net_seq_num %" PRIu64 " refcount %ld"
+				" stream %" PRIu64 " trace %" PRIu64
+				" session %" PRIu64,
+				index,
+				index->index_n.key,
+				stream->ref.refcount,
+				index->stream->stream_handle,
+				index->stream->trace->id,
+				index->stream->trace->session->id);
+	}
+	rcu_read_unlock();
+}
+
 void print_relay_streams(void)
 {
 	struct lttng_ht_iter iter;
@@ -434,6 +455,7 @@ void print_relay_streams(void)
 				stream->stream_handle,
 				stream->trace->id,
 				stream->trace->session->id);
+		print_stream_indexes(stream);
 		stream_put(stream);
 	}
 	rcu_read_unlock();
Signature · 1 line
2.1.4
#5 lore raw

Re: [PATCH lttng-tools 1/5] Fix: LPOLLHUP and LPOLLERR when there is still data in pipe/socket

In reply to a message not in the archive

Merged in master and stable-2.7.

Thanks!
Jérémie

Quoted 981 lines · Mathieu Desnoyers

The event mask returned by poll/epoll is a bitwise mask made of all the events observed. On bidirectional sockets, there are cases where combinations of LPOLLHUP/LPOLLERR and LPOLLIN/LPOLLPRI can be raised at the same time.

Currently the overall behavior in sessiond, consumerd and relayd is to handle LPOLLHUP or LPOLLERR immediately, whether or not there is still data to read in the socket. Unfortunately, this behavior may discard the last information made available on the pipe or socket.

Audit all uses of LPOLLHUP and LPOLLERR on sockets on which we expect data to ensure that we deal with LPOLLIN or LPOLLPRI, and catch the hangup when read or recvmsg returns 0. Keep the LPOLLHUP and LPOLLERR handling, but only when LPOLLIN is not raised, just in case some unforeseen error happens when sending the reply.

This is one correct case where we can handle LPOLLHUP and LPOLLERR directly without caring about LPOLLIN: sockets where we are expected to write and then read the reply (e.g. command sockets). It is then OK for a dedicated thread to watch for LPOLLHUP and LPOLLERR.

---
 src/bin/lttng-consumerd/health-consumerd.c |   3 +-
 src/bin/lttng-relayd/health-relayd.c       |   7 +-
 src/bin/lttng-relayd/live.c                |  36 +++++----
 src/bin/lttng-relayd/main.c                |  97 ++++++++++++++---------
 src/bin/lttng-sessiond/agent-thread.c      |  45 ++++++-----
 src/bin/lttng-sessiond/ht-cleanup.c        |  43 +++++-----
 src/bin/lttng-sessiond/main.c              | 102 ++++++++++++++++--------
 src/bin/lttng-sessiond/ust-thread.c        |  91 +++++++++++-----------
 src/common/consumer.c                      | 121 +++++++++++++++++------------
 9 files changed, 319 insertions(+), 226 deletions(-)
src/bin/lttng-consumerd/health-consumerd.c +2 -1
index fc9a266..5be1c97 100644
@@ -275,7 +275,8 @@ restart:
 
                        /* Event on the registration socket */
                        if (pollfd == sock) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)
+                                               && !(revents & LPOLLIN)) {
                                        ERR("Health socket poll error");
                                        goto error;
                                }
src/bin/lttng-relayd/health-relayd.c +6 -1
index d34a376..be0b0b1 100644
@@ -346,9 +346,14 @@ restart:
 
                        /* Event on the registration socket */
                        if (pollfd == sock) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & LPOLLIN) {
+                                       continue;
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                        ERR("Health socket poll error");
                                        goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        }
                }
src/bin/lttng-relayd/live.c +23 -13
index a1fbbbe..8da2d62 100644
@@ -542,10 +542,7 @@ restart:
                                goto exit;
                        }
 
-                       if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                               ERR("socket poll error");
-                               goto error;
-                       } else if (revents & LPOLLIN) {
+                       if (revents & LPOLLIN) {
                                /*
                                 * A new connection is requested, therefore a
                                 * viewer connection is allocated in this
@@ -588,6 +585,12 @@ restart:
                                 * exchange in cds_wfcq_enqueue.
                                 */
                                futex_nto1_wake(&viewer_conn_queue.futex);
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               ERR("socket poll error");
+                               goto error;
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
@@ -1908,10 +1911,7 @@ restart:
 
                        /* Inspect the relay conn pipe for new connection. */
                        if (pollfd == live_conn_pipe[0]) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       ERR("Relay live pipe error");
-                                       goto error;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        struct relay_connection *conn;
 
                                        ret = lttng_read(live_conn_pipe[0],
@@ -1923,6 +1923,12 @@ restart:
                                                        LPOLLIN | LPOLLRDHUP);
                                        connection_ht_add(viewer_connections_ht, conn);
                                        DBG("Connection socket %d added to poll", conn->sock->fd);
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       ERR("Relay live pipe error");
+                                       goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        } else {
                                /* Connection activity. */
@@ -1933,11 +1939,7 @@ restart:
                                        continue;
                                }
 
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       cleanup_connection_pollfd(&events, pollfd);
-                                       /* Put "create" ownership reference. */
-                                       connection_put(conn);
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        ret = conn->sock->ops->recvmsg(conn->sock, &recv_hdr,
                                                        sizeof(recv_hdr), 0);
                                        if (ret <= 0) {
@@ -1956,6 +1958,14 @@ restart:
                                                        DBG("Viewer connection closed with %d", pollfd);
                                                }
                                        }
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       cleanup_connection_pollfd(&events, pollfd);
+                                       /* Put "create" ownership reference. */
+                                       connection_put(conn);
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       connection_put(conn);
+                                       goto error;
                                }
                                /* Put local "get_by_sock" reference. */
                                connection_put(conn);
src/bin/lttng-relayd/main.c +59 -38
index 2ce9bf0..dcaaaa8 100644
@@ -864,10 +864,7 @@ restart:
                                goto exit;
                        }
 
-                       if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                               ERR("socket poll error");
-                               goto error;
-                       } else if (revents & LPOLLIN) {
+                       if (revents & LPOLLIN) {
                                /*
                                 * A new connection is requested, therefore a
                                 * sessiond/consumerd connection is allocated in
@@ -919,6 +916,12 @@ restart:
                                 * exchange in cds_wfcq_enqueue.
                                 */
                                futex_nto1_wake(&relay_conn_queue.futex);
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               ERR("socket poll error");
+                               goto error;
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
@@ -2429,10 +2432,7 @@ restart:
 
                        /* Inspect the relay conn pipe for new connection */
                        if (pollfd == relay_conn_pipe[0]) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       ERR("Relay connection pipe error");
-                                       goto error;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        struct relay_connection *conn;
 
                                        ret = lttng_read(relay_conn_pipe[0], &conn, sizeof(conn));
@@ -2443,6 +2443,12 @@ restart:
                                                        LPOLLIN | LPOLLRDHUP);
                                        connection_ht_add(relay_connections_ht, conn);
                                        DBG("Connection socket %d added", conn->sock->fd);
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       ERR("Relay connection pipe error");
+                                       goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        } else {
                                struct relay_connection *ctrl_conn;
@@ -2451,29 +2457,8 @@ restart:
                                /* If not found, there is a synchronization issue. */
                                assert(ctrl_conn);
 
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       relay_thread_close_connection(&events, pollfd, ctrl_conn);
-                                       if (last_seen_data_fd == pollfd) {
-                                               last_seen_data_fd = last_notdel_data_fd;
-                                       }
-                               } else if (revents & LPOLLIN) {
-                                       if (ctrl_conn->type == RELAY_CONTROL) {
-                                               ret = ctrl_conn->sock->ops->recvmsg(ctrl_conn->sock, &recv_hdr,
-                                                               sizeof(recv_hdr), 0);
-                                               if (ret <= 0) {
-                                                       /* Connection closed */
-                                                       relay_thread_close_connection(&events, pollfd,
-                                                               ctrl_conn);
-                                               } else {
-                                                       ret = relay_process_control(&recv_hdr, ctrl_conn);
-                                                       if (ret < 0) {
-                                                               /* Clear the session on error. */
-                                                               relay_thread_close_connection(&events, pollfd,
-                                                                       ctrl_conn);
-                                                       }
-                                                       seen_control = 1;
-                                               }
-                                       } else {
+                               if (ctrl_conn->type == RELAY_DATA) {
+                                       if (revents & LPOLLIN) {
                                                /*
                                                 * Flag the last seen data fd not deleted. It will be
                                                 * used as the last seen fd if any fd gets deleted in
@@ -2481,9 +2466,39 @@ restart:
                                                 */
                                                last_notdel_data_fd = pollfd;
                                        }
+                                       goto put_ctrl_connection;
+                               }
+                               assert(ctrl_conn->type == RELAY_CONTROL);
+
+                               if (revents & LPOLLIN) {
+                                       ret = ctrl_conn->sock->ops->recvmsg(ctrl_conn->sock,
+                                                       &recv_hdr, sizeof(recv_hdr), 0);
+                                       if (ret <= 0) {
+                                               /* Connection closed */
+                                               relay_thread_close_connection(&events, pollfd,
+                                                               ctrl_conn);
+                                       } else {
+                                               ret = relay_process_control(&recv_hdr, ctrl_conn);
+                                               if (ret < 0) {
+                                                       /* Clear the session on error. */
+                                                       relay_thread_close_connection(&events,
+                                                                       pollfd, ctrl_conn);
+                                               }
+                                               seen_control = 1;
+                                       }
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       relay_thread_close_connection(&events,
+                                                       pollfd, ctrl_conn);
+                                       if (last_seen_data_fd == pollfd) {
+                                               last_seen_data_fd = last_notdel_data_fd;
+                                       }
                                } else {
-                                       ERR("Unknown poll events %u for sock %d", revents, pollfd);
+                                       ERR("Unexpected poll events %u for control sock %d",
+                                                       revents, pollfd);
+                                       connection_put(ctrl_conn);
+                                       goto error;
                                }
+                       put_ctrl_connection:
                                connection_put(ctrl_conn);
                        }
                }
@@ -2533,17 +2548,17 @@ restart:
                                /* Skip it. Might be removed before. */
                                continue;
                        }
+                       if (data_conn->type == RELAY_CONTROL) {
+                               goto put_data_connection;
+                       }
+                       assert(data_conn->type == RELAY_DATA);
 
                        if (revents & LPOLLIN) {
-                               if (data_conn->type != RELAY_DATA) {
-                                       goto put_connection;
-                               }
-
                                ret = relay_process_data(data_conn);
                                /* Connection closed */
                                if (ret < 0) {
                                        relay_thread_close_connection(&events, pollfd,
-                                               data_conn);
+                                                       data_conn);
                                        /*
                                         * Every goto restart call sets the last seen fd where
                                         * here we don't really care since we gracefully
@@ -2555,8 +2570,14 @@ restart:
                                        connection_put(data_conn);
                                        goto restart;
                                }
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               relay_thread_close_connection(&events, pollfd,
+                                               data_conn);
+                       } else {
+                               ERR("Unknown poll events %u for data sock %d",
+                                               revents, pollfd);
                        }
-               put_connection:
+               put_data_connection:
                        connection_put(data_conn);
                }
                last_seen_data_fd = -1;
src/bin/lttng-sessiond/agent-thread.c +24 -21
index d1bb122..2b6f776 100644
@@ -296,35 +296,22 @@ restart:
                                goto exit;
                        }
 
-                       /*
-                        * Check first if this is a POLLERR since POLLIN is also included
-                        * in an error value thus checking first.
-                        */
-                       if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                               /* Removing from the poll set */
-                               ret = lttng_poll_del(&events, pollfd);
-                               if (ret < 0) {
-                                       goto error;
-                               }
-
-                               agent_destroy_app_by_sock(pollfd);
-                       } else if (revents & (LPOLLIN)) {
+                       if (revents & LPOLLIN) {
                                int new_fd;
                                struct agent_app *app = NULL;
 
-                               /* Pollin event of agent app socket should NEVER happen. */
                                assert(pollfd == reg_sock->fd);
-
                                new_fd = handle_registration(reg_sock, &app);
                                if (new_fd < 0) {
-                                       WARN("[agent-thread] agent registration failed. Ignoring.");
-                                       /* Somehow the communication failed. Just continue. */
                                        continue;
                                }
                                /* Should not have a NULL app on success. */
                                assert(app);
 
-                               /* Only add poll error event to only detect shutdown. */
+                               /*
+                                * Since this is a command socket (write then read),
+                                * only add poll error event to only detect shutdown.
+                                */
                                ret = lttng_poll_add(&events, new_fd,
                                                LPOLLERR | LPOLLHUP | LPOLLRDHUP);
                                if (ret < 0) {
@@ -336,10 +323,26 @@ restart:
                                update_agent_app(app);
 
                                /* On failure, the poll will detect it and clean it up. */
-                               (void) agent_send_registration_done(app);
+                               ret = agent_send_registration_done(app);
+                               if (ret < 0) {
+                                       /* Removing from the poll set */
+                                       ret = lttng_poll_del(&events, new_fd);
+                                       if (ret < 0) {
+                                               goto error;
+                                       }
+                                       agent_destroy_app_by_sock(new_fd);
+                                       continue;
+                               }
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               /* Removing from the poll set */
+                               ret = lttng_poll_del(&events, pollfd);
+                               if (ret < 0) {
+                                       goto error;
+                               }
+                               agent_destroy_app_by_sock(pollfd);
                        } else {
-                               ERR("Unknown poll events %u for sock %d", revents, pollfd);
-                               continue;
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
src/bin/lttng-sessiond/ht-cleanup.c +21 -22
index c5f64da..b5f58d0 100644
@@ -103,32 +103,31 @@ restart:
                                continue;
                        }
 
-                       if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                       if (revents & LPOLLIN) {
+                               /* Get socket from dispatch thread. */
+                               size_ret = lttng_read(ht_cleanup_pipe[0], &ht,
+                                               sizeof(ht));
+                               if (size_ret < sizeof(ht)) {
+                                       PERROR("ht cleanup notify pipe");
+                                       goto error;
+                               }
+                               health_code_update();
+                               /*
+                                * The whole point of this thread is to call
+                                * lttng_ht_destroy from a context that is NOT:
+                                * 1) a read-side RCU lock,
+                                * 2) a call_rcu thread.
+                                */
+                               lttng_ht_destroy(ht);
+
+                               health_code_update();
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                ERR("ht cleanup pipe error");
                                goto error;
-                       } else if (!(revents & LPOLLIN)) {
-                               /* No POLLIN and not a catched error, stop the thread. */
-                               ERR("ht cleanup failed. revent: %u", revents);
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
                                goto error;
                        }
-
-                       /* Get socket from dispatch thread. */
-                       size_ret = lttng_read(ht_cleanup_pipe[0], &ht,
-                                       sizeof(ht));
-                       if (size_ret < sizeof(ht)) {
-                               PERROR("ht cleanup notify pipe");
-                               goto error;
-                       }
-                       health_code_update();
-                       /*
-                        * The whole point of this thread is to call
-                        * lttng_ht_destroy from a context that is NOT:
-                        * 1) a read-side RCU lock,
-                        * 2) a call_rcu thread.
-                        */
-                       lttng_ht_destroy(ht);
-
-                       health_code_update();
                }
 
                for (i = 0; i < nb_fd; i++) {
src/bin/lttng-sessiond/main.c +68 -34
index 0e7de6e..13bd649 100644
@@ -1165,31 +1165,33 @@ static void *thread_manage_kernel(void *data)
                        }
 
                        /* Check for data on kernel pipe */
-                       if (pollfd == kernel_poll_pipe[0] && (revents & LPOLLIN)) {
-                               (void) lttng_read(kernel_poll_pipe[0],
-                                       &tmp, 1);
-                               /*
-                                * Ret value is useless here, if this pipe gets any actions an
-                                * update is required anyway.
-                                */
-                               update_poll_flag = 1;
-                               continue;
-                       } else {
-                               /*
-                                * New CPU detected by the kernel. Adding kernel stream to
-                                * kernel session and updating the kernel consumer
-                                */
-                               if (revents & LPOLLIN) {
+                       if (revents & LPOLLIN) {
+                               if (pollfd == kernel_poll_pipe[0]) {
+                                       (void) lttng_read(kernel_poll_pipe[0],
+                                               &tmp, 1);
+                                       /*
+                                        * Ret value is useless here, if this pipe gets any actions an
+                                        * update is required anyway.
+                                        */
+                                       update_poll_flag = 1;
+                                       continue;
+                               } else {
+                                       /*
+                                        * New CPU detected by the kernel. Adding kernel stream to
+                                        * kernel session and updating the kernel consumer
+                                        */
                                        ret = update_kernel_stream(&kconsumer_data, pollfd);
                                        if (ret < 0) {
                                                continue;
                                        }
                                        break;
-                                       /*
-                                        * TODO: We might want to handle the LPOLLERR | LPOLLHUP
-                                        * and unregister kernel stream at this point.
-                                        */
                                }
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               update_poll_flag = 1;
+                               continue;
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
@@ -1319,9 +1321,14 @@ restart:
 
                /* Event on the registration socket */
                if (pollfd == consumer_data->err_sock) {
-                       if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                       if (revents & LPOLLIN) {
+                               continue;
+                       } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                ERR("consumer err socket poll error");
                                goto error;
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
@@ -1451,7 +1458,8 @@ restart_poll:
 
                        if (pollfd == sock) {
                                /* Event on the consumerd socket */
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)
+                                               && !(revents & LPOLLIN)) {
                                        ERR("consumer err socket second poll error");
                                        goto error;
                                }
@@ -1469,6 +1477,11 @@ restart_poll:
 
                                goto exit;
                        } else if (pollfd == consumer_data->metadata_fd) {
+                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)
+                                               && !(revents & LPOLLIN)) {
+                                       ERR("consumer err metadata socket second poll error");
+                                       goto error;
+                               }
                                /* UST metadata requests */
                                ret = ust_consumer_metadata_request(
                                                &consumer_data->metadata_sock);
@@ -1636,10 +1649,7 @@ static void *thread_manage_apps(void *data)
 
                        /* Inspect the apps cmd pipe */
                        if (pollfd == apps_cmd_pipe[0]) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       ERR("Apps command pipe error");
-                                       goto error;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        int sock;
 
                                        /* Empty pipe */
@@ -1652,9 +1662,8 @@ static void *thread_manage_apps(void *data)
                                        health_code_update();
 
                                        /*
-                                        * We only monitor the error events of the socket. This
-                                        * thread does not handle any incoming data from UST
-                                        * (POLLIN).
+                                        * Since this is a command socket (write then read),
+                                        * we only monitor the error events of the socket.
                                         */
                                        ret = lttng_poll_add(&events, sock,
                                                        LPOLLERR | LPOLLHUP | LPOLLRDHUP);
@@ -1663,6 +1672,12 @@ static void *thread_manage_apps(void *data)
                                        }
 
                                        DBG("Apps with sock %d added to poll set", sock);
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       ERR("Apps command pipe error");
+                                       goto error;
+                               } else {
+                                       ERR("Unknown poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        } else {
                                /*
@@ -1678,6 +1693,9 @@ static void *thread_manage_apps(void *data)
 
                                        /* Socket closed on remote end. */
                                        ust_app_unregister(pollfd);
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        }
 
@@ -1825,6 +1843,9 @@ static void sanitize_wait_queue(struct ust_reg_wait_queue *wait_queue)
                                 */
                                wait_node = NULL;
                                break;
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               goto error;
                        }
                }
        }
@@ -2185,10 +2206,7 @@ static void *thread_registration_apps(void *data)
 
                        /* Event on the registration socket */
                        if (pollfd == apps_sock) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
-                                       ERR("Register apps socket poll error");
-                                       goto error;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        sock = lttcomm_accept_unix_sock(apps_sock);
                                        if (sock < 0) {
                                                goto error;
@@ -2275,6 +2293,12 @@ static void *thread_registration_apps(void *data)
                                         * barrier with the exchange in cds_wfcq_enqueue.
                                         */
                                        futex_nto1_wake(&ust_cmd_queue.futex);
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                                       ERR("Register apps socket poll error");
+                                       goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        }
                }
@@ -4177,9 +4201,14 @@ restart:
 
                        /* Event on the registration socket */
                        if (pollfd == sock) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & LPOLLIN) {
+                                       continue;
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                        ERR("Health socket poll error");
                                        goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        }
                }
@@ -4354,9 +4383,14 @@ static void *thread_manage_clients(void *data)
 
                        /* Event on the registration socket */
                        if (pollfd == client_sock) {
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & LPOLLIN) {
+                                       continue;
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                        ERR("Client socket poll error");
                                        goto error;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                        }
                }
src/bin/lttng-sessiond/ust-thread.c +45 -46
index b421eb2..9c944c5 100644
@@ -57,7 +57,8 @@ void *ust_thread_manage_notify(void *data)
        }
 
        /* Add notify pipe to the pollset. */
-       ret = lttng_poll_add(&events, apps_cmd_notify_pipe[0], LPOLLIN | LPOLLERR);
+       ret = lttng_poll_add(&events, apps_cmd_notify_pipe[0],
+                       LPOLLIN | LPOLLERR | LPOLLHUP | LPOLLRDHUP);
        if (ret < 0) {
                goto error;
        }
@@ -109,45 +110,56 @@ restart:
                        if (pollfd == apps_cmd_notify_pipe[0]) {
                                int sock;
 
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & LPOLLIN) {
+                                       /* Get socket from dispatch thread. */
+                                       size_ret = lttng_read(apps_cmd_notify_pipe[0],
+                                                       &sock, sizeof(sock));
+                                       if (size_ret < sizeof(sock)) {
+                                               PERROR("read apps notify pipe");
+                                               goto error;
+                                       }
+                                       health_code_update();
+
+                                       ret = lttng_poll_add(&events, sock,
+                                                       LPOLLIN | LPOLLERR | LPOLLHUP | LPOLLRDHUP);
+                                       if (ret < 0) {
+                                               /*
+                                                * It's possible we've reached the max poll fd allowed.
+                                                * Let's close the socket but continue normal execution.
+                                                */
+                                               ret = close(sock);
+                                               if (ret) {
+                                                       PERROR("close notify socket %d", sock);
+                                               }
+                                               lttng_fd_put(LTTNG_FD_APPS, 1);
+                                               continue;
+                                       }
+                                       DBG3("UST thread notify added sock %d to pollset", sock);
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                        ERR("Apps notify command pipe error");
                                        goto error;
-                               } else if (!(revents & LPOLLIN)) {
-                                       /* No POLLIN and not a catched error, stop the thread. */
-                                       ERR("Notify command pipe failed. revent: %u", revents);
-                                       goto error;
-                               }
-
-                               /* Get socket from dispatch thread. */
-                               size_ret = lttng_read(apps_cmd_notify_pipe[0],
-                                               &sock, sizeof(sock));
-                               if (size_ret < sizeof(sock)) {
-                                       PERROR("read apps notify pipe");
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
                                        goto error;
                                }
-                               health_code_update();
-
-                               ret = lttng_poll_add(&events, sock,
-                                               LPOLLIN | LPOLLERR | LPOLLHUP | LPOLLRDHUP);
-                               if (ret < 0) {
-                                       /*
-                                        * It's possible we've reached the max poll fd allowed.
-                                        * Let's close the socket but continue normal execution.
-                                        */
-                                       ret = close(sock);
-                                       if (ret) {
-                                               PERROR("close notify socket %d", sock);
-                                       }
-                                       lttng_fd_put(LTTNG_FD_APPS, 1);
-                                       continue;
-                               }
-                               DBG3("UST thread notify added sock %d to pollset", sock);
                        } else {
                                /*
                                 * At this point, we know that a registered application
                                 * triggered the event.
                                 */
-                               if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
+                               if (revents & (LPOLLIN | LPOLLPRI)) {
+                                       ret = ust_app_recv_notify(pollfd);
+                                       if (ret < 0) {
+                                               /* Removing from the poll set */
+                                               ret = lttng_poll_del(&events, pollfd);
+                                               if (ret < 0) {
+                                                       goto error;
+                                               }
+
+                                               /* The socket is closed after a grace period here. */
+                                               ust_app_notify_sock_unregister(pollfd);
+                                       }
+                               } else if (revents & (LPOLLERR | LPOLLHUP | LPOLLRDHUP)) {
                                        /* Removing from the poll set */
                                        ret = lttng_poll_del(&events, pollfd);
                                        if (ret < 0) {
@@ -156,22 +168,9 @@ restart:
 
                                        /* The socket is closed after a grace period here. */
                                        ust_app_notify_sock_unregister(pollfd);
-                               } else if (revents & (LPOLLIN | LPOLLPRI)) {
-                                       ret = ust_app_recv_notify(pollfd);
-                                       if (ret < 0) {
-                                               /*
-                                                * If the notification failed either the application is
-                                                * dead or an internal error happened. In both cases,
-                                                * we can only continue here. If the application is
-                                                * dead, an unregistration will follow or else the
-                                                * application will notice that we are not responding
-                                                * on that socket and will close it.
-                                                */
-                                               continue;
-                                       }
                                } else {
-                                       ERR("Unknown poll events %u for sock %d", revents, pollfd);
-                                       continue;
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto error;
                                }
                                health_code_update();
                        }
src/common/consumer.c +71 -51
index c34f47c..c8628e8 100644
@@ -2240,26 +2240,22 @@ restart:
                        }
 
                        if (pollfd == lttng_pipe_get_readfd(ctx->consumer_metadata_pipe)) {
-                               if (revents & (LPOLLERR | LPOLLHUP )) {
-                                       DBG("Metadata thread pipe hung up");
-                                       /*
-                                        * Remove the pipe from the poll set and continue the loop
-                                        * since their might be data to consume.
-                                        */
-                                       lttng_poll_del(&events,
-                                                       lttng_pipe_get_readfd(ctx->consumer_metadata_pipe));
-                                       lttng_pipe_read_close(ctx->consumer_metadata_pipe);
-                                       continue;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        ssize_t pipe_len;
 
                                        pipe_len = lttng_pipe_read(ctx->consumer_metadata_pipe,
                                                        &stream, sizeof(stream));
                                        if (pipe_len < sizeof(stream)) {
-                                               PERROR("read metadata stream");
+                                               if (pipe_len < 0) {
+                                                       PERROR("read metadata stream");
+                                               }
                                                /*
-                                                * Continue here to handle the rest of the streams.
+                                                * Remove the pipe from the poll set and continue the loop
+                                                * since their might be data to consume.
                                                 */
+                                               lttng_poll_del(&events,
+                                                               lttng_pipe_get_readfd(ctx->consumer_metadata_pipe));
+                                               lttng_pipe_read_close(ctx->consumer_metadata_pipe);
                                                continue;
                                        }
 
@@ -2276,6 +2272,19 @@ restart:
                                        /* Add metadata stream to the global poll events list */
                                        lttng_poll_add(&events, stream->wait_fd,
                                                        LPOLLIN | LPOLLPRI | LPOLLHUP);
+                               } else if (revents & (LPOLLERR | LPOLLHUP)) {
+                                       DBG("Metadata thread pipe hung up");
+                                       /*
+                                        * Remove the pipe from the poll set and continue the loop
+                                        * since their might be data to consume.
+                                        */
+                                       lttng_poll_del(&events,
+                                                       lttng_pipe_get_readfd(ctx->consumer_metadata_pipe));
+                                       lttng_pipe_read_close(ctx->consumer_metadata_pipe);
+                                       continue;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto end;
                                }
 
                                /* Handle other stream */
@@ -2294,8 +2303,30 @@ restart:
                        stream = caa_container_of(node, struct lttng_consumer_stream,
                                        node);
 
-                       /* Check for error event */
-                       if (revents & (LPOLLERR | LPOLLHUP)) {
+                       if (revents & (LPOLLIN | LPOLLPRI)) {
+                               /* Get the data out of the metadata file descriptor */
+                               DBG("Metadata available on fd %d", pollfd);
+                               assert(stream->wait_fd == pollfd);
+
+                               do {
+                                       health_code_update();
+
+                                       len = ctx->on_buffer_ready(stream, ctx);
+                                       /*
+                                        * We don't check the return value here since if we get
+                                        * a negative len, it means an error occured thus we
+                                        * simply remove it from the poll set and free the
+                                        * stream.
+                                        */
+                               } while (len > 0);
+
+                               /* It's ok to have an unavailable sub-buffer */
+                               if (len < 0 && len != -EAGAIN && len != -ENODATA) {
+                                       /* Clean up stream from consumer and free it. */
+                                       lttng_poll_del(&events, stream->wait_fd);
+                                       consumer_del_metadata_stream(stream, metadata_ht);
+                               }
+                       } else if (revents & (LPOLLERR | LPOLLHUP)) {
                                DBG("Metadata fd %d is hup|err.", pollfd);
                                if (!stream->hangup_flush_done
                                                && (consumer_data.type == LTTNG_CONSUMER32_UST
@@ -2323,31 +2354,11 @@ restart:
                                 * and securely free the stream.
                                 */
                                consumer_del_metadata_stream(stream, metadata_ht);
-                       } else if (revents & (LPOLLIN | LPOLLPRI)) {
-                               /* Get the data out of the metadata file descriptor */
-                               DBG("Metadata available on fd %d", pollfd);
-                               assert(stream->wait_fd == pollfd);
-
-                               do {
-                                       health_code_update();
-
-                                       len = ctx->on_buffer_ready(stream, ctx);
-                                       /*
-                                        * We don't check the return value here since if we get
-                                        * a negative len, it means an error occured thus we
-                                        * simply remove it from the poll set and free the
-                                        * stream.
-                                        */
-                               } while (len > 0);
-
-                               /* It's ok to have an unavailable sub-buffer */
-                               if (len < 0 && len != -EAGAIN && len != -ENODATA) {
-                                       /* Clean up stream from consumer and free it. */
-                                       lttng_poll_del(&events, stream->wait_fd);
-                                       consumer_del_metadata_stream(stream, metadata_ht);
-                               }
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               rcu_read_unlock;
+                               goto end;
                        }
-
                        /* Release RCU lock for the stream looked up */
                        rcu_read_unlock();
                }
@@ -2812,21 +2823,16 @@ restart:
                        }
 
                        if (pollfd == ctx->consumer_channel_pipe[0]) {
-                               if (revents & (LPOLLERR | LPOLLHUP)) {
-                                       DBG("Channel thread pipe hung up");
-                                       /*
-                                        * Remove the pipe from the poll set and continue the loop
-                                        * since their might be data to consume.
-                                        */
-                                       lttng_poll_del(&events, ctx->consumer_channel_pipe[0]);
-                                       continue;
-                               } else if (revents & LPOLLIN) {
+                               if (revents & LPOLLIN) {
                                        enum consumer_channel_action action;
                                        uint64_t key;
 
                                        ret = read_channel_pipe(ctx, &chan, &key, &action);
                                        if (ret <= 0) {
-                                               ERR("Error reading channel pipe");
+                                               if (ret < 0) {
+                                                       ERR("Error reading channel pipe");
+                                               }
+                                               lttng_poll_del(&events, ctx->consumer_channel_pipe[0]);
                                                continue;
                                        }
 
@@ -2843,7 +2849,7 @@ restart:
                                                rcu_read_unlock();
                                                /* Add channel to the global poll events list */
                                                lttng_poll_add(&events, chan->wait_fd,
-                                                               LPOLLIN | LPOLLPRI);
+                                                               LPOLLERR | LPOLLHUP);
                                                break;
                                        case CONSUMER_CHANNEL_DEL:
                                        {
@@ -2903,6 +2909,17 @@ restart:
                                                ERR("Unknown action");
                                                break;
                                        }
+                               } else if (revents & (LPOLLERR | LPOLLHUP)) {
+                                       DBG("Channel thread pipe hung up");
+                                       /*
+                                        * Remove the pipe from the poll set and continue the loop
+                                        * since their might be data to consume.
+                                        */
+                                       lttng_poll_del(&events, ctx->consumer_channel_pipe[0]);
+                                       continue;
+                               } else {
+                                       ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                                       goto end;
                                }
 
                                /* Handle other stream */
@@ -2941,6 +2958,10 @@ restart:
                                                && !uatomic_read(&chan->nb_init_stream_left)) {
                                        consumer_del_channel(chan);
                                }
+                       } else {
+                               ERR("Unexpected poll events %u for sock %d", revents, pollfd);
+                               rcu_read_unlock();
+                               goto end;
                        }
 
                        /* Release RCU lock for the channel looked up */
--

2.1.4

Signature · 8 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): [email protected]

#7 lore raw

Re: [PATCH lttng-tools 2/5] Fix: relayd: handle consumerd crashes without leak

Merged in master and stable-2.7.

Thanks!
Jérémie

Quoted 156 lines · Mathieu Desnoyers

We can be clever about indexes partially received in cases where we received the data socket part, but not the control socket part: since we're currently closing the stream on behalf of the control socket, we *know* there won't be any more control information for this socket. Therefore, we can destroy all indexes for which we have received only the file descriptor (from data socket). This takes care of consumerd crashes between sending the data and control information for a packet. Since those are sent in that order, we take care of consumerd crashes.

---
 src/bin/lttng-relayd/index.c  | 39 +++++++++++++++++++++++++++
 src/bin/lttng-relayd/index.h  |  2 ++
 src/bin/lttng-relayd/stream.c | 61 +++++++++++++++++++++++++++++++++----------
 3 files changed, 88 insertions(+), 14 deletions(-)
src/bin/lttng-relayd/index.c +39 -0
index 7182e36..cb7ae3d 100644
@@ -333,3 +333,42 @@ void relay_index_close_all(struct relay_stream *stream)
        }
        rcu_read_unlock();
 }
+
+void relay_index_close_partial_fd(struct relay_stream *stream)
+{
+       struct lttng_ht_iter iter;
+       struct relay_index *index;
+
+       rcu_read_lock();
+       cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter,
+                       index, index_n.node) {
+               if (!index->index_fd) {
+                       continue;
+               }
+               /*
+                * Partial index has its index_fd: we have only
+                * received its info from the data socket.
+                * Put self-ref from index.
+                */
+               relay_index_put(index);
+       }
+       rcu_read_unlock();
+}
+
+uint64_t relay_index_find_last(struct relay_stream *stream)
+{
+       struct lttng_ht_iter iter;
+       struct relay_index *index;
+       uint64_t net_seq_num = -1ULL;
+
+       rcu_read_lock();
+       cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter,
+                       index, index_n.node) {
+               if (net_seq_num == -1ULL ||
+                               index->index_n.key > net_seq_num) {
+                       net_seq_num = index->index_n.key;
+               }
+       }
+       rcu_read_unlock();
+       return net_seq_num;
+}
src/bin/lttng-relayd/index.h +2 -0
index e882ed9..15c4ac8 100644
@@ -71,5 +71,7 @@ int relay_index_set_data(struct relay_index *index,
 int relay_index_try_flush(struct relay_index *index);
 
 void relay_index_close_all(struct relay_stream *stream);
+void relay_index_close_partial_fd(struct relay_stream *stream);
+uint64_t relay_index_find_last(struct relay_stream *stream);
 
 #endif /* _RELAY_INDEX_H */
src/bin/lttng-relayd/stream.c +47 -15
index 2a59d1e..cac8763 100644
@@ -354,26 +354,59 @@ void try_stream_close(struct relay_stream *stream)
        }
 
        stream->close_requested = true;
-       /*
-        * We shortcut the data pending check if no bound is known for this
-        * stream. This prevents us from never closing the stream in the case
-        * where a connection would be closed before a "close" command has
-        * been received.
-        *
-        * TODO
-        * This still leaves open the question of handling missing data after
-        * a bound has been set by a stream close command. Since we have no
-        * way of pairing data and control connection, and that a data
-        * connection has no ownership of a stream, it is likely that a
-        * timeout approach would be appropriate to handle dangling streams.
-        */
+
+       if (stream->last_net_seq_num == -1ULL) {
+               /*
+                * Handle connection close without explicit stream close
+                * command.
+                *
+                * We can be clever about indexes partially received in
+                * cases where we received the data socket part, but not
+                * the control socket part: since we're currently closing
+                * the stream on behalf of the control socket, we *know*
+                * there won't be any more control information for this
+                * socket. Therefore, we can destroy all indexes for
+                * which we have received only the file descriptor (from
+                * data socket). This takes care of consumerd crashes
+                * between sending the data and control information for
+                * a packet. Since those are sent in that order, we take
+                * care of consumerd crashes.
+                */
+               relay_index_close_partial_fd(stream);
+               /*
+                * Use the highest net_seq_num we currently have pending
+                * As end of stream indicator.  Leave last_net_seq_num
+                * at -1ULL if we cannot find any index.
+                */
+               stream->last_net_seq_num = relay_index_find_last(stream);
+               /* Fall-through into the next check. */
+       }
+
        if (stream->last_net_seq_num != -1ULL &&
                        ((int64_t) (stream->prev_seq - stream->last_net_seq_num)) < 0) {
-               /* Don't close since we still have data pending. */
+               /*
+                * Don't close since we still have data pending. This
+                * handles cases where an explicit close command has
+                * been received for this stream, and cases where the
+                * connection has been closed, and we are awaiting for
+                * index information from the data socket. It is
+                * therefore expected that all the index fd information
+                * we need has already been received on the control
+                * socket. Matching index information from data socket
+                * should be Expected Soon(TM).
+                *
+                * TODO: We should implement a timer to garbage collect
+                * streams after a timeout to be resilient against a
+                * consumerd implementation that would not match this
+                * expected behavior.
+                */
                pthread_mutex_unlock(&stream->lock);
                DBG("closing stream %" PRIu64 " aborted since it still has data pending", stream->stream_handle);
                return;
        }
+       /*
+        * We received all the indexes we can expect.
+        */
        stream_unpublish(stream);
        stream->closed = true;
        /* Relay indexes are only used by the "consumer/sessiond" end. */
--

2.1.4

Signature · 8 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): [email protected]

#9 lore raw

Not merged, as discussed, since closing the stream will set net_seq_idx to -1ULL and that the relayd will remain available in the ht even if we encounter a relayd connection error.

Jérémie

Quoted 37 lines · Mathieu Desnoyers

When consumer_stream_write_index cannot find the relay daemon, it currently assumes that the reason for this situation is that it should write to a file. Add a check to see if indeed we should write to a file, of if this is caused by a removed relayd. Return an error and print an error to the console in this situation.

---
 src/common/consumer-stream.c | 8 ++++++++
 1 file changed, 8 insertions(+)
src/common/consumer-stream.c +8 -1
index 604e2f4..23cc882 100644
@@ -374,10 +374,18 @@ int consumer_stream_write_index(struct lttng_consumer_stream *stream,
                pthread_mutex_lock(&relayd->ctrl_sock_mutex);
                ret = relayd_send_index(&relayd->control_sock, index,
                                stream->relayd_stream_id, stream->next_net_seq_num - 1);
+
                pthread_mutex_unlock(&relayd->ctrl_sock_mutex);
        } else {
                ssize_t size_ret;
 
+               if (stream->net_seq_idx != -1ULL) {
+                       ERR("Cannot find relayd for write index, net_seq_idx %" PRIu64,
+                               stream->net_seq_idx);
+                       ret = -1;
+                       goto error;
+               }
+
                size_ret = index_write(stream->index_fd, index,
                                sizeof(struct ctf_packet_index));
                if (size_ret < sizeof(struct ctf_packet_index)) {
--

2.1.4

Signature · 8 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): [email protected]

#11 lore raw

Re: [PATCH lttng-tools 4/5] Enhance relayd error reporting

Merged in master and stable-2.7.

Thanks!
Jérémie

Quoted 62 lines · Mathieu Desnoyers

relay_process_data has error cases that don't print any error to the console. Add those cases, and enhance the information provided by error output within handle_index_data().

---
 src/bin/lttng-relayd/main.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
src/bin/lttng-relayd/main.c +9 -3
index dcaaaa8..0c7ef8d 100644
@@ -2084,8 +2084,8 @@ static int handle_index_data(struct relay_stream *stream, uint64_t net_seq_num,
        /* Get data offset because we are about to update the index. */
        data_offset = htobe64(stream->tracefile_size_current);
 
-       DBG("handle_index_data: stream %" PRIu64 " data offset %" PRIu64,
-               stream->stream_handle, stream->tracefile_size_current);
+       DBG("handle_index_data: stream %" PRIu64 " net_seq_num %" PRIu64 " data offset %" PRIu64,
+                       stream->stream_handle, net_seq_num, stream->tracefile_size_current);
 
        /*
         * Lookup for an existing index for that stream id/sequence
@@ -2183,6 +2183,7 @@ static int relay_process_data(struct relay_connection *conn)
        stream_id = be64toh(data_hdr.stream_id);
        stream = stream_get_by_id(stream_id);
        if (!stream) {
+               ERR("relay_process_data: Cannot find stream %" PRIu64, stream_id);
                ret = -1;
                goto end;
        }
@@ -2212,6 +2213,8 @@ static int relay_process_data(struct relay_connection *conn)
                if (ret == 0) {
                        /* Orderly shutdown. Not necessary to print an error. */
                        DBG("Socket %d did an orderly shutdown", conn->sock->fd);
+               } else {
+                       ERR("Socket %d error %d", conn->sock->fd, ret);
                }
                ret = -1;
                goto end_stream_put;
@@ -2255,6 +2258,8 @@ static int relay_process_data(struct relay_connection *conn)
        if (session->minor >= 4 && !session->snapshot) {
                ret = handle_index_data(stream, net_seq_num, rotate_index);
                if (ret < 0) {
+                       ERR("handle_index_data: fail stream %" PRIu64 " net_seq_num %" PRIu64 " ret %d",
+                                       stream->stream_handle, net_seq_num, ret);
                        goto end_stream_unlock;
                }
        }
@@ -2273,6 +2278,8 @@ static int relay_process_data(struct relay_connection *conn)
        ret = write_padding_to_file(stream->stream_fd->fd,
                        be32toh(data_hdr.padding_size));
        if (ret < 0) {
+               ERR("write_padding_to_file: fail stream %" PRIu64 " net_seq_num %" PRIu64 " ret %d",
+                               stream->stream_handle, net_seq_num, ret);
                goto end_stream_unlock;
        }
        stream->tracefile_size_current +=
--

2.1.4

Signature · 8 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

A message that is not in the archive (sent off-list or not yet received): [email protected]

#13 lore raw

Re: [PATCH lttng-tools 5/5] Print relayd stream indexes

Merged in master and stable-2.7.

Thanks!
Jérémie

Quoted 51 lines · Mathieu Desnoyers

In debug output, print leaking stream index information when exiting from relay daemon.

---
 src/bin/lttng-relayd/stream.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
src/bin/lttng-relayd/stream.c +22 -1
index cac8763..b604919 100644
@@ -416,6 +416,27 @@ void try_stream_close(struct relay_stream *stream)
        stream_put(stream);
 }
 
+static void print_stream_indexes(struct relay_stream *stream)
+{
+       struct lttng_ht_iter iter;
+       struct relay_index *index;
+
+       rcu_read_lock();
+       cds_lfht_for_each_entry(stream->indexes_ht->ht, &iter.iter, index,
+                       index_n.node) {
+               DBG("index %p net_seq_num %" PRIu64 " refcount %ld"
+                               " stream %" PRIu64 " trace %" PRIu64
+                               " session %" PRIu64,
+                               index,
+                               index->index_n.key,
+                               stream->ref.refcount,
+                               index->stream->stream_handle,
+                               index->stream->trace->id,
+                               index->stream->trace->session->id);
+       }
+       rcu_read_unlock();
+}
+
 void print_relay_streams(void)
 {
        struct lttng_ht_iter iter;
@@ -434,6 +455,7 @@ void print_relay_streams(void)
                                stream->stream_handle,
                                stream->trace->id,
                                stream->trace->session->id);
+               print_stream_indexes(stream);
                stream_put(stream);
        }
        rcu_read_unlock();
--

2.1.4

Signature · 8 lines
Jérémie Galarneau
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
[email protected]
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev