Skip to contentSkip to conversation

perf kvm stat live: cache mmap()ed events

RFC PATCH In review Alexander Yarygin 4 participants 6 messages Sep 12, 2014 – Sep 15, 2014 perf events

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 perf kvm stat live: cache mmap()ed events

During mmap() process 'perf kvm stat live' gets a pointer to events and passes them to the session queue. Events are stored in shared memory and eventually they will be overwritten by the kernel. The problem is, that when events come too fast, old events can be overwritten before they have been processed that can lead to perf crash.

To prevent that happening, we can copy upcoming events and pass a copy to the session queue. There is a safe place to copy event: before perf_evlist__mmap_consume() is executed. There are 3 places to free it:
when event is processed, when it's lost and on exit, if it's turned out unprocessed.

Cc Arnaldo Carvalho de Melo <[email protected]>
Cc Christian Borntraeger <[email protected]>
Cc David Ahern <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Jiri Olsa <[email protected]>
Cc Paul Mackerras <[email protected]>
Cc Peter Zijlstra <[email protected]>
---
 tools/perf/builtin-kvm.c   | 88 +++++++++++++++++++++++++++++++++++++++++-----
 tools/perf/util/kvm-stat.h |  2 ++
 2 files changed, 81 insertions(+), 9 deletions(-)
tools/perf/builtin-kvm.c +79 -9
index 43367eb..183e613 100644
@@ -625,6 +625,62 @@ static void print_result(struct perf_kvm_stat *kvm)
 }
 
 #ifdef HAVE_TIMERFD_SUPPORT
+
+struct events_cache {
+	union perf_event event;
+	struct list_head list;
+};
+
+static void init_events_cache(struct perf_kvm_stat *kvm)
+{
+	kvm->cache = malloc(sizeof(struct events_cache));
+	INIT_LIST_HEAD(&kvm->cache->list);
+}
+
+static union perf_event *alloc_event_cache(struct events_cache *cache,
+					   union perf_event *event)
+{
+	struct events_cache *new = malloc(sizeof(struct events_cache));
+
+	memmove(&new->event, event, event->header.size);
+	list_add(&new->list, &cache->list);
+
+	return &new->event;
+}
+
+static void free_event_cache(struct events_cache *cache,
+			     union perf_event *event)
+{
+	struct list_head *p, *p2;
+
+	list_for_each_safe(p, p2, &cache->list) {
+		struct events_cache *entry;
+
+		entry = list_entry(p, struct events_cache, list);
+
+		if (&entry->event == event) {
+			list_del(&entry->list);
+			free(entry);
+
+			break;
+		}
+	}
+}
+
+static void clear_events_cache(struct events_cache *cache)
+{
+	struct list_head *p, *p2;
+
+	list_for_each_safe(p, p2, &cache->list) {
+		struct event_cache *entry;
+
+		entry = list_entry(p, struct events_cache, list);
+		list_del(&entry->list);
+		free(entry);
+	}
+	free(cache);
+}
+
 static int process_lost_event(struct perf_tool *tool,
 			      union perf_event *event __maybe_unused,
 			      struct perf_sample *sample __maybe_unused,
@@ -633,6 +689,10 @@ static int process_lost_event(struct perf_tool *tool,
 	struct perf_kvm_stat *kvm = container_of(tool, struct perf_kvm_stat, tool);
 
 	kvm->lost_events++;
+
+	if (kvm->live)
+		free_event_cache(kvm->cache, event);
+
 	return 0;
 }
 #endif
@@ -655,6 +715,7 @@ static int process_sample_event(struct perf_tool *tool,
 	struct thread *thread;
 	struct perf_kvm_stat *kvm = container_of(tool, struct perf_kvm_stat,
 						 tool);
+	int err;
 
 	if (skip_sample(kvm, sample))
 		return 0;
@@ -663,13 +724,20 @@ static int process_sample_event(struct perf_tool *tool,
 	if (thread == NULL) {
 		pr_debug("problem processing %d event, skipping it.\n",
 			event->header.type);
-		return -1;
+
+		err = -1;
+		goto out;
 	}
 
-	if (!handle_kvm_event(kvm, thread, evsel, sample))
-		return -1;
+	err = !handle_kvm_event(kvm, thread, evsel, sample);
 
-	return 0;
+out:
+#ifdef HAVE_TIMERFD_SUPPORT
+	if (kvm->live)
+		free_event_cache(kvm->cache, event);
+#endif
+
+	return err;
 }
 
 static int cpu_isa_config(struct perf_kvm_stat *kvm)
@@ -732,15 +800,15 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx,
 			return -1;
 		}
 
-		err = perf_session_queue_event(kvm->session, event, &sample, 0);
-		/*
-		 * FIXME: Here we can't consume the event, as perf_session_queue_event will
-		 *        point to it, and it'll get possibly overwritten by the kernel.
-		 */
+		event = alloc_event_cache(kvm->cache, event);
+
 		perf_evlist__mmap_consume(kvm->evlist, idx);
 
+		err = perf_session_queue_event(kvm->session, event, &sample, 0);
+
 		if (err) {
 			pr_err("Failed to enqueue sample: %d\n", err);
+			free_event_cache(kvm->cache, event);
 			return -1;
 		}
 
@@ -959,6 +1027,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm)
 
 	/* everything is good - enable the events and process */
 	perf_evlist__enable(kvm->evlist);
+	init_events_cache(kvm);
 
 	while (!done) {
 		int rc;
@@ -978,6 +1047,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm)
 			err = poll(pollfds, nr_fds, 100);
 	}
 
+	clear_events_cache(kvm->cache);
 	perf_evlist__disable(kvm->evlist);
 
 	if (err == 0) {
tools/perf/util/kvm-stat.h +2 -0
index 0b5a8cd..856dbf7 100644
@@ -97,6 +97,8 @@ struct perf_kvm_stat {
 
 	struct rb_root result;
 
+	struct events_cache *cache;
+
 	int timerfd;
 	unsigned int display_time;
 	bool live;

--
1.9.1

#1 lore raw

On Fri, Sep 12, 2014 at 08:27:43PM +0400, Alexander Yarygin wrote:

SNIP

Quoted 11 lines
tools/perf/builtin-kvm.c +11 -0
+
+	list_for_each_safe(p, p2, &cache->list) {
+		struct events_cache *entry;
+
+		entry = list_entry(p, struct events_cache, list);
+
+		if (&entry->event == event) {
+			list_del(&entry->list);
+			free(entry);
+
+			break;

tools/perf/builtin-kvm.c static void print_result(struct perf_kvm_stat *kvm) · line 675

Quoted 10 lines
tools/perf/builtin-kvm.c +10 -0
+		}
+	}
+}
+
+static void clear_events_cache(struct events_cache *cache)
+{
+	struct list_head *p, *p2;
+
+	list_for_each_safe(p, p2, &cache->list) {
+		struct event_cache *entry;

s/event_cache/events_cache/ ? does not compile for me otherwise

Quoted 7 lines
tools/perf/builtin-kvm.c +7 -0
+
+		entry = list_entry(p, struct events_cache, list);
+		list_del(&entry->list);
+		free(entry);
+	}
+	free(cache);
+}

SNIP

Quoted 7 lines

}

static int cpu_isa_config(struct perf_kvm_stat *kvm)

tools/perf/builtin-kvm.c +0 -0
@@ -732,15 +800,15 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx,
 			return -1;
 		}
 

tools/perf/builtin-kvm.c static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx, · line 807

Quoted 10 lines
tools/perf/builtin-kvm.c +3 -5
-		err = perf_session_queue_event(kvm->session, event, &sample, 0);
-		/*
-		 * FIXME: Here we can't consume the event, as perf_session_queue_event will
-		 *        point to it, and it'll get possibly overwritten by the kernel.
-		 */
+		event = alloc_event_cache(kvm->cache, event);
+
 		perf_evlist__mmap_consume(kvm->evlist, idx);
 
+		err = perf_session_queue_event(kvm->session, event, &sample, 0);

got conflict here, this got recently changed.. needs to have tool param:

- err = perf_session_queue_event(kvm->session, event, &sample, 0);
+ err = perf_session_queue_event(kvm->session, event, &kvm->tool, &sample, 0);

jirka

#2 lore raw
In reply to Alexander Yarygin
Quoted 11 lines · Alexander Yarygin

During mmap() process 'perf kvm stat live' gets a pointer to events and passes them to the session queue. Events are stored in shared memory and eventually they will be overwritten by the kernel. The problem is, that when events come too fast, old events can be overwritten before they have been processed that can lead to perf crash.

To prevent that happening, we can copy upcoming events and pass a copy to the session queue. There is a safe place to copy event: before perf_evlist__mmap_consume() is executed. There are 3 places to free it:
when event is processed, when it's lost and on exit, if it's turned out unprocessed.

Did you see what I proposed a year ago:
https://lkml.org/lkml/2013/9/6/388

The intent is to keep the copy generic and not local a command since conceptually other live commands need the same.

David

#3 lore raw
Quoted 13 lines · David Ahern
Quoted 11 lines · Alexander Yarygin

During mmap() process 'perf kvm stat live' gets a pointer to events and passes them to the session queue. Events are stored in shared memory and eventually they will be overwritten by the kernel. The problem is, that when events come too fast, old events can be overwritten before they have been processed that can lead to perf crash.

To prevent that happening, we can copy upcoming events and pass a copy to the session queue. There is a safe place to copy event: before perf_evlist__mmap_consume() is executed. There are 3 places to free it:
when event is processed, when it's lost and on exit, if it's turned out unprocessed.

Quoted 7 lines

Did you see what I proposed a year ago:
https://lkml.org/lkml/2013/9/6/388

The intent is to keep the copy generic and not local a command since conceptually other live commands need the same.

David

Hello,

Yes, your patch works fine. But as far as I understand, right now only the 'perf kvm stat live' is infected: other 'live' tools were fixed by the patch "PERF: The tail position of the event buffer should only be modified after actually use that event." and since they don't use ordered queue they don't need a copying. That's why I came up with 'perf kvm stat live' specific approach. Maybe I missed something...

Anyhow, having >30K events is a quite usual situation on s390 and the 'perf kvm stat live' command hardly works there, so it would be good to have at least some working solution. Any ideas? :)

Thanks.

#4 lore raw
Quoted 23 lines · Alexander Yarygin
Quoted 13 lines · David Ahern
Quoted 11 lines · Alexander Yarygin

During mmap() process 'perf kvm stat live' gets a pointer to events and passes them to the session queue. Events are stored in shared memory and eventually they will be overwritten by the kernel. The problem is, that when events come too fast, old events can be overwritten before they have been processed that can lead to perf crash.

To prevent that happening, we can copy upcoming events and pass a copy to the session queue. There is a safe place to copy event: before perf_evlist__mmap_consume() is executed. There are 3 places to free it:
when event is processed, when it's lost and on exit, if it's turned out unprocessed.

Quoted 7 lines

Did you see what I proposed a year ago:
https://lkml.org/lkml/2013/9/6/388

The intent is to keep the copy generic and not local a command since conceptually other live commands need the same.

David

Quoted 8 lines

Hello,

Yes, your patch works fine. But as far as I understand, right now only the 'perf kvm stat live' is infected: other 'live' tools were fixed by the patch "PERF: The tail position of the event buffer should only be modified after actually use that event." and since they don't use ordered queue they don't need a copying. That's why I came up with 'perf kvm stat live' specific approach. Maybe I missed something...

No, you understand the problem. My point is that the key issue with the code (overwriting events in the mmap'ed buffers) really has nothing to do with perf-kvm. It might be the only command in-tree today but others could come along, so might as well fix the issue in the event processing code.

Quoted 4 lines

Anyhow, having >30K events is a quite usual situation on s390 and the 'perf kvm stat live' command hardly works there, so it would be good to have at least some working solution. Any ideas? :)

Sure. With nested VMs on x86 I see 100,000's of events per second. That use case is what drove me to look at copying events. I just have not had time to come back to that one.

David

#5 lore raw
In reply to Alexander Yarygin

Em Mon, Sep 15, 2014 at 04:57:34PM +0400, Alexander Yarygin escreveu:

Quoted 27 lines
Quoted 13 lines · David Ahern
Quoted 11 lines · Alexander Yarygin

During mmap() process 'perf kvm stat live' gets a pointer to events and passes them to the session queue. Events are stored in shared memory and eventually they will be overwritten by the kernel. The problem is, that when events come too fast, old events can be overwritten before they have been processed that can lead to perf crash.

To prevent that happening, we can copy upcoming events and pass a copy to the session queue. There is a safe place to copy event: before perf_evlist__mmap_consume() is executed. There are 3 places to free it:
when event is processed, when it's lost and on exit, if it's turned out unprocessed.

Quoted 5 lines

Did you see what I proposed a year ago:
https://lkml.org/lkml/2013/9/6/388

The intent is to keep the copy generic and not local a command since conceptually other live commands need the same.

Yes, your patch works fine. But as far as I understand, right now only the 'perf kvm stat live' is infected: other 'live' tools were fixed by the patch "PERF: The tail position of the event buffer should only be modified after actually use that event." and since they don't use ordered queue they don't need a copying. That's why I came up with 'perf kvm stat live' specific approach. Maybe I missed something...

Quoted 3 lines

Anyhow, having >30K events is a quite usual situation on s390 and the 'perf kvm stat live' command hardly works there, so it would be good to have at least some working solution. Any ideas? :)

I don't have a problem solving this on some tool where the problem happens and then, when some other tool hits the same problem, then one looks at existing tools, figures out if some tool-specific solution can be reused, moves it to common code, possibly improving it in the process, and reuses it.

Sometimes trying to make it general from day one may lead to overengineering :-)

With that said, David's patch looks clean and small enough, so if that solves your problem, please let me know if I can apply it and your Tested-by/Any-other-tags, ok?

- Arnaldo