Skip to contentSkip to conversation

perf top: Move the timeout warning from event processing thread to display thread

RFC PATCH New [email protected] 1 participant 1 message Oct 30, 2018 perf events

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 perf top: Move the timeout warning from event processing thread to display thread

From: Kan Liang <[email protected]>

The main event processing thread may hang if the ring buffer event processing timeouts.

Analysis from David Miller:
"It hangs the event thread, because the ui call waits for a keypress but the display thread will eat them up and the event thread thus hangs in select()."

The timeout warning is moved to display thread.

The nr_rb_read is introduced to track the times of perf_top__mmap_read(), which is the main function of event processing. If the nr_rb_read doesn't increase during the refresh time, the display thread may output stale data. The timeout warning will be triggered.

The timeout warning can only be triggered one time to avoid the annoying and duplicated warning message.

The first perf_top__mmap_read() is moved to after display thread create. Because the perf_top__mmap_read() could cost long time. For example, the function may cost tens of minutes on Knights Landing platform with parallel kernel build. There will be nothing displayed on the screen. The display thread has to be created before perf_top__mmap_read(). But at that time, the data is not ready. Display thread has to sleep refresh time.

Fix: 8cc42de736b6 ("perf top: Check the latency of perf_top__mmap_read()")

---
 tools/perf/builtin-c2c.c       |  4 +--
 tools/perf/builtin-report.c    |  3 ++-
 tools/perf/builtin-top.c       | 39 +++++++++++++++++++---------
 tools/perf/ui/browsers/hists.c | 58 ++++++++++++++++++++++++++++++++++--------
 tools/perf/ui/browsers/hists.h |  2 +-
 tools/perf/util/hist.h         |  6 +++--
 tools/perf/util/top.h          |  1 +
 7 files changed, 85 insertions(+), 28 deletions(-)
tools/perf/builtin-c2c.c +2 -2
index f3aa9d0..1e77515 100644
@@ -2371,7 +2371,7 @@ static int perf_c2c__browse_cacheline(struct hist_entry *he)
 	c2c_browser__update_nr_entries(browser);
 
 	while (1) {
-		key = hist_browser__run(browser, "? - help", true);
+		key = hist_browser__run(browser, "? - help", true, NULL);
 
 		switch (key) {
 		case 's':
@@ -2440,7 +2440,7 @@ static int perf_c2c__hists_browse(struct hists *hists)
 	c2c_browser__update_nr_entries(browser);
 
 	while (1) {
-		key = hist_browser__run(browser, "? - help", true);
+		key = hist_browser__run(browser, "? - help", true, NULL);
 
 		switch (key) {
 		case 'q':
tools/perf/builtin-report.c +2 -1
index 257c9c1..2fc1273 100644
@@ -561,7 +561,8 @@ static int report__browse_hists(struct report *rep)
 		ret = perf_evlist__tui_browse_hists(evlist, help, NULL,
 						    rep->min_percent,
 						    &session->header.env,
-						    true, &rep->annotation_opts);
+						    true, &rep->annotation_opts,
+						    NULL);
 		/*
 		 * Usually "ret" is the last pressed key, and we only
 		 * care if the key notifies us to switch data file.
tools/perf/builtin-top.c +27 -12
index d21d875..95409de 100644
@@ -584,6 +584,8 @@ static void *display_thread_tui(void *arg)
 		.refresh	= top->delay_secs,
 	};
 
+	sleep(top->delay_secs);
+
 	/* In order to read symbols from other namespaces perf to  needs to call
 	 * setns(2).  This isn't permitted if the struct_fs has multiple users.
 	 * unshare(2) the fs so that we may continue to setns into namespaces
@@ -607,7 +609,8 @@ static void *display_thread_tui(void *arg)
 				      top->min_percent,
 				      &top->session->header.env,
 				      !top->record_opts.overwrite,
-				      &top->annotation_opts);
+				      &top->annotation_opts,
+				      &top->nr_rb_read);
 
 	done = 1;
 	return NULL;
@@ -633,6 +636,11 @@ static void *display_thread(void *arg)
 	struct termios save;
 	struct perf_top *top = arg;
 	int delay_msecs, c;
+	bool rb_read_timeout_warned = false;
+	bool rb_read_timeout = false;
+	int last_nr_rb_read = 0;
+
+	sleep(top->delay_secs);
 
 	/* In order to read symbols from other namespaces perf to  needs to call
 	 * setns(2).  This isn't permitted if the struct_fs has multiple users.
@@ -651,12 +659,26 @@ static void *display_thread(void *arg)
 
 	while (!done) {
 		perf_top__print_sym_table(top);
+
+		if (!rb_read_timeout_warned && rb_read_timeout) {
+			color_fprintf(stdout, PERF_COLOR_RED,
+				      "Too slow to read ring buffer.\n"
+				      "Please try increasing the period (-c) or\n"
+				      "decreasing the freq (-F) or\n"
+				      "limiting the number of CPUs (-C)\n");
+			rb_read_timeout_warned = true;
+		}
 		/*
 		 * Either timeout expired or we got an EINTR due to SIGWINCH,
 		 * refresh screen in both cases.
 		 */
 		switch (poll(&stdin_poll, 1, delay_msecs)) {
 		case 0:
+			if (atomic_read(&top->nr_rb_read) == last_nr_rb_read)
+				rb_read_timeout = true;
+			else
+				rb_read_timeout = false;
+			last_nr_rb_read = atomic_read(&top->nr_rb_read);
 			continue;
 		case -1:
 			if (errno == EINTR)
@@ -881,10 +903,10 @@ static void perf_top__mmap_read(struct perf_top *top)
 {
 	bool overwrite = top->record_opts.overwrite;
 	struct perf_evlist *evlist = top->evlist;
-	unsigned long long start, end;
 	int i;
 
-	start = rdclock();
+	atomic_inc(&top->nr_rb_read);
+
 	if (overwrite)
 		perf_evlist__toggle_bkw_mmap(evlist, BKW_MMAP_DATA_PENDING);
 
@@ -895,13 +917,6 @@ static void perf_top__mmap_read(struct perf_top *top)
 		perf_evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
 		perf_evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING);
 	}
-	end = rdclock();
-
-	if ((end - start) > (unsigned long long)top->delay_secs * NSEC_PER_SEC)
-		ui__warning("Too slow to read ring buffer.\n"
-			    "Please try increasing the period (-c) or\n"
-			    "decreasing the freq (-F) or\n"
-			    "limiting the number of CPUs (-C)\n");
 }
 
 /*
@@ -1137,8 +1152,6 @@ static int __cmd_top(struct perf_top *top)
 	/* Wait for a minimal set of events before starting the snapshot */
 	perf_evlist__poll(top->evlist, 100);
 
-	perf_top__mmap_read(top);
-
 	ret = -1;
 	if (pthread_create(&thread, NULL, (use_browser > 0 ? display_thread_tui :
 							    display_thread), top)) {
@@ -1146,6 +1159,8 @@ static int __cmd_top(struct perf_top *top)
 		goto out_delete;
 	}
 
+	perf_top__mmap_read(top);
+
 	if (top->realtime_prio) {
 		struct sched_param param;
 
tools/perf/ui/browsers/hists.c +48 -10

145 lines not on this page · raw message

tools/perf/ui/browsers/hists.h +1 -1
index 91d3e18..af6c1b9 100644
@@ -32,7 +32,7 @@ struct hist_browser {
 struct hist_browser *hist_browser__new(struct hists *hists);
 void hist_browser__delete(struct hist_browser *browser);
 int hist_browser__run(struct hist_browser *browser, const char *help,
-		      bool warn_lost_event);
+		      bool warn_lost_event, atomic_t *nr_rb_read);
 void hist_browser__init(struct hist_browser *browser,
 			struct hists *hists);
 #endif /* _PERF_UI_BROWSER_HISTS_H_ */
tools/perf/util/hist.h +4 -2
index 3badd7f..67f31163 100644
@@ -442,7 +442,8 @@ int perf_evlist__tui_browse_hists(struct perf_evlist *evlist, const char *help,
 				  float min_pcnt,
 				  struct perf_env *env,
 				  bool warn_lost_event,
-				  struct annotation_options *annotation_options);
+				  struct annotation_options *annotation_options,
+				  atomic_t *nr_rb_read);
 int script_browse(const char *script_opt);
 #else
 static inline
@@ -452,7 +453,8 @@ int perf_evlist__tui_browse_hists(struct perf_evlist *evlist __maybe_unused,
 				  float min_pcnt __maybe_unused,
 				  struct perf_env *env __maybe_unused,
 				  bool warn_lost_event __maybe_unused,
-				  struct annotation_options *annotation_options __maybe_unused)
+				  struct annotation_options *annotation_options __maybe_unused,
+				  atomic_t *nr_rb_read __maybe_unused)
 {
 	return 0;
 }
tools/perf/util/top.h +1 -0
index 9add1f7..2869d9d 100644
@@ -40,6 +40,7 @@ struct perf_top {
 	const char	   *sym_filter;
 	float		   min_percent;
 	unsigned int	   nr_threads_synthesize;
+	atomic_t	   nr_rb_read;
 };
 
 #define CONSOLE_CLEAR ""
Signature · 1 line
2.7.4