Skip to contentSkip to conversation

perf tools: Support reading PERF_FORMAT_LOST

PATCH In review Namhyung Kim 3 participants 9 messages Aug 15, 2022 – Aug 16, 2022 perf events

lore
Share & tools
lore

Namhyung Kim

author maintainer
#0 lore raw

cover perf tools: Support reading PERF_FORMAT_LOST

Hello,

The kernel v6.0 added PERF_FORMAT_LOST which can read a number of lost samples for the given event. As it can change the output format of read(2) and perf sample data, it needs to access them carefully.

You can get the code from 'perf/read-lost-v1' brach on

git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git

Thanks,
Namhyung

Namhyung Kim (4):
tools headers UAPI: Sync linux/perf_event.h with the kernel sources tools lib perf: Handle read format in perf_evsel__read() tools lib perf: Add a test case for read formats
perf tools: Support reading PERF_FORMAT_LOST

 tools/include/uapi/linux/perf_event.h         |   5 +-
 tools/lib/perf/evsel.c                        |  72 ++++++++
 tools/lib/perf/include/perf/event.h           |   3 +-
 tools/lib/perf/include/perf/evsel.h           |   4 +-
 tools/lib/perf/tests/test-evsel.c             | 161 ++++++++++++++++++
 tools/perf/tests/sample-parsing.c             |  14 +-
 tools/perf/util/event.h                       |  18 +-
 tools/perf/util/evsel.c                       |  33 +++-
 .../scripting-engines/trace-event-python.c    |  16 +-
 tools/perf/util/session.c                     |  32 ++--
 tools/perf/util/synthetic-events.c            |  34 +++-
 11 files changed, 354 insertions(+), 38 deletions(-)

base-commit: 568035b01cfb107af8d2e4bd2fb9aea22cf5b868

Signature · 1 line
2.37.1.595.g718a3a8f04-goog
author maintainer
#1 lore raw

PATCH 1/4 tools headers UAPI: Sync linux/perf_event.h with the kernel sources

To pick the trivial change in:

119a784c8127 ("perf/core: Add a new read format to get a number of lost samples")

---
 tools/include/uapi/linux/perf_event.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
tools/include/uapi/linux/perf_event.h +4 -1
index e2b77fbca91e..581ed4bdc062 100644
@@ -301,6 +301,7 @@ enum {
  *	  { u64		time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED
  *	  { u64		time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
  *	  { u64		id;           } && PERF_FORMAT_ID
+ *	  { u64		lost;         } && PERF_FORMAT_LOST
  *	} && !PERF_FORMAT_GROUP
  *
  *	{ u64		nr;
@@ -308,6 +309,7 @@ enum {
  *	  { u64		time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING
  *	  { u64		value;
  *	    { u64	id;           } && PERF_FORMAT_ID
+ *	    { u64	lost;         } && PERF_FORMAT_LOST
  *	  }		cntr[nr];
  *	} && PERF_FORMAT_GROUP
  * };
@@ -317,8 +319,9 @@ enum perf_event_read_format {
 	PERF_FORMAT_TOTAL_TIME_RUNNING		= 1U << 1,
 	PERF_FORMAT_ID				= 1U << 2,
 	PERF_FORMAT_GROUP			= 1U << 3,
+	PERF_FORMAT_LOST			= 1U << 4,
 
-	PERF_FORMAT_MAX = 1U << 4,		/* non-ABI */
+	PERF_FORMAT_MAX = 1U << 5,		/* non-ABI */
 };
 
 #define PERF_ATTR_SIZE_VER0	64	/* sizeof first published struct */
Signature · 1 line
2.37.1.595.g718a3a8f04-goog
author maintainer
#2 lore raw

PATCH 2/4 tools lib perf: Handle read format in perf_evsel__read()

In reply to Namhyung Kim

The perf_counts_values should be increased to read the new lost data. Also adjust values after read according the read format.

This supports PERF_FORMAT_GROUP which has a different data format but it's only available for leader events. Currently it doesn't have an API to read sibling (member) events in the group. But users may read the sibling event directly.

Also reading from mmap would be disabled when the read format has ID or LOST bit as it's not exposed via mmap.

---
 tools/lib/perf/evsel.c              | 72 +++++++++++++++++++++++++++++
 tools/lib/perf/include/perf/event.h |  3 +-
 tools/lib/perf/include/perf/evsel.h |  4 +-
 3 files changed, 77 insertions(+), 2 deletions(-)
tools/lib/perf/evsel.c +72 -0
index 952f3520d5c2..fc23670231cb 100644
@@ -305,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	if (read_format & PERF_FORMAT_ID)
 		entry += sizeof(u64);
 
+	if (read_format & PERF_FORMAT_LOST)
+		entry += sizeof(u64);
+
 	if (read_format & PERF_FORMAT_GROUP) {
 		nr = evsel->nr_members;
 		size += sizeof(u64);
@@ -314,24 +317,93 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	return size;
 }
 
+/* This only reads values for the leader */
+static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
+				  int thread, struct perf_counts_values *count)
+{
+	size_t size = perf_evsel__read_size(evsel);
+	int *fd = FD(evsel, cpu_map_idx, thread);
+	u64 read_format = evsel->attr.read_format;
+	u64 *data;
+	int idx = 1;
+
+	if (fd == NULL || *fd < 0)
+		return -EINVAL;
+
+	data = calloc(1, size);
+	if (data == NULL)
+		return -ENOMEM;
+
+	if (readn(*fd, data, size) <= 0) {
+		free(data);
+		return -errno;
+	}
+
+	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		count->ena = data[idx++];
+	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		count->run = data[idx++];
+
+	/* value is always available */
+	count->val = data[idx++];
+	if (read_format & PERF_FORMAT_ID)
+		count->id = data[idx++];
+	if (read_format & PERF_FORMAT_LOST)
+		count->lost = data[idx++];
+
+	free(data);
+	return 0;
+}
+
+/*
+ * The perf read format is very flexible.  It needs to set the proper
+ * values according to the read format.
+ */
+static void perf_evsel__adjust_values(struct perf_evsel *evsel,
+				      struct perf_counts_values *count)
+{
+	u64 read_format = evsel->attr.read_format;
+
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)) {
+		memmove(&count->values[2], &count->values[1], 24);
+		count->ena = 0;
+	}
+
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)) {
+		memmove(&count->values[3], &count->values[2], 16);
+		count->run = 0;
+	}
+
+	if (!(read_format & PERF_FORMAT_ID)) {
+		memmove(&count->values[4], &count->values[3], 8);
+		count->id = 0;
+	}
+}
+
 int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
 		     struct perf_counts_values *count)
 {
 	size_t size = perf_evsel__read_size(evsel);
 	int *fd = FD(evsel, cpu_map_idx, thread);
+	u64 read_format = evsel->attr.read_format;
 
 	memset(count, 0, sizeof(*count));
 
 	if (fd == NULL || *fd < 0)
 		return -EINVAL;
 
+	if (read_format & PERF_FORMAT_GROUP)
+		return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
+
 	if (MMAP(evsel, cpu_map_idx, thread) &&
+	    !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
 	    !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
 		return 0;
 
 	if (readn(*fd, count->values, size) <= 0)
 		return -errno;
 
+	perf_evsel__adjust_values(evsel, count);
 	return 0;
 }
 
tools/lib/perf/include/perf/event.h +2 -1
index 556bb06798f2..38dd35cbca71 100644
@@ -76,7 +76,7 @@ struct perf_record_lost_samples {
 };
 
 /*
- * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID
+ * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID | PERF_FORMAT_LOST
  */
 struct perf_record_read {
 	struct perf_event_header header;
@@ -85,6 +85,7 @@ struct perf_record_read {
 	__u64			 time_enabled;
 	__u64			 time_running;
 	__u64			 id;
+	__u64			 lost;
 };
 
 struct perf_record_throttle {
tools/lib/perf/include/perf/evsel.h +3 -1
index 699c0ed97d34..6f92204075c2 100644
@@ -18,8 +18,10 @@ struct perf_counts_values {
 			uint64_t val;
 			uint64_t ena;
 			uint64_t run;
+			uint64_t id;
+			uint64_t lost;
 		};
-		uint64_t values[3];
+		uint64_t values[5];
 	};
 };
 
Signature · 1 line
2.37.1.595.g718a3a8f04-goog
#3 lore raw

Re: [PATCH 2/4] tools lib perf: Handle read format in perf_evsel__read()

Quoted 47 lines · Namhyung Kim

The perf_counts_values should be increased to read the new lost data. Also adjust values after read according the read format.

This supports PERF_FORMAT_GROUP which has a different data format but it's only available for leader events. Currently it doesn't have an API to read sibling (member) events in the group. But users may read the sibling event directly.

Also reading from mmap would be disabled when the read format has ID or LOST bit as it's not exposed via mmap.

---
 tools/lib/perf/evsel.c              | 72 +++++++++++++++++++++++++++++
 tools/lib/perf/include/perf/event.h |  3 +-
 tools/lib/perf/include/perf/evsel.h |  4 +-
 3 files changed, 77 insertions(+), 2 deletions(-)
tools/lib/perf/evsel.c +14 -0
index 952f3520d5c2..fc23670231cb 100644
@@ -305,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	if (read_format & PERF_FORMAT_ID)
 		entry += sizeof(u64);
 
+	if (read_format & PERF_FORMAT_LOST)
+		entry += sizeof(u64);
+
 	if (read_format & PERF_FORMAT_GROUP) {
 		nr = evsel->nr_members;
 		size += sizeof(u64);
@@ -314,24 +317,93 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	return size;
 }
 
+/* This only reads values for the leader */
+static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
+				  int thread, struct perf_counts_values *count)
+{
+	size_t size = perf_evsel__read_size(evsel);
+	int *fd = FD(evsel, cpu_map_idx, thread);
+	u64 read_format = evsel->attr.read_format;
+	u64 *data;
+	int idx = 1;
+
+	if (fd == NULL || *fd < 0)

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 340

Quoted 10 lines
tools/lib/perf/evsel.c +10 -0
+		return -EINVAL;
+
+	data = calloc(1, size);
+	if (data == NULL)
+		return -ENOMEM;
+
+	if (readn(*fd, data, size) <= 0) {
+		free(data);
+		return -errno;
+	}

could you please put in here some comment that this is intentionaly reading only the leader or better yet rename the function? I was lost before I got to read the changelog ;-)

Quoted 31 lines
tools/lib/perf/evsel.c +31 -0
+
+	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		count->ena = data[idx++];
+	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		count->run = data[idx++];
+
+	/* value is always available */
+	count->val = data[idx++];
+	if (read_format & PERF_FORMAT_ID)
+		count->id = data[idx++];
+	if (read_format & PERF_FORMAT_LOST)
+		count->lost = data[idx++];
+
+	free(data);
+	return 0;
+}
+
+/*
+ * The perf read format is very flexible.  It needs to set the proper
+ * values according to the read format.
+ */
+static void perf_evsel__adjust_values(struct perf_evsel *evsel,
+				      struct perf_counts_values *count)
+{
+	u64 read_format = evsel->attr.read_format;
+
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)) {
+		memmove(&count->values[2], &count->values[1], 24);
+		count->ena = 0;
+	}
+

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 381

Quoted 10 lines
tools/lib/perf/evsel.c +10 -0
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)) {
+		memmove(&count->values[3], &count->values[2], 16);
+		count->run = 0;
+	}
+
+	if (!(read_format & PERF_FORMAT_ID)) {
+		memmove(&count->values[4], &count->values[3], 8);
+		count->id = 0;
+	}
+}

could we do this the same way we read group counters.. like make read into local buffer and initialize perf_counts_values values based on format, something like:

        readn(fd, data ...

        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
                count->ena = data[idx++];
        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
                count->run = data[idx++];

        /* value is always available */
        count->val = data[idx++];
        if (read_format & PERF_FORMAT_ID)
                count->id = data[idx++];
        if (read_format & PERF_FORMAT_LOST)
                count->lost = data[idx++];

and perhaps we should cancel that perf_counts_values's union and keep only 'val/ena/run...' fields?

jirka

#4 lore raw

Re: [PATCH 2/4] tools lib perf: Handle read format in perf_evsel__read()

Em Tue, Aug 16, 2022 at 03:19:20PM +0200, Jiri Olsa escreveu:

Quoted 110 lines · Namhyung Kim
Quoted 47 lines

The perf_counts_values should be increased to read the new lost data. Also adjust values after read according the read format.

This supports PERF_FORMAT_GROUP which has a different data format but it's only available for leader events. Currently it doesn't have an API to read sibling (member) events in the group. But users may read the sibling event directly.

Also reading from mmap would be disabled when the read format has ID or LOST bit as it's not exposed via mmap.

---
 tools/lib/perf/evsel.c              | 72 +++++++++++++++++++++++++++++
 tools/lib/perf/include/perf/event.h |  3 +-
 tools/lib/perf/include/perf/evsel.h |  4 +-
 3 files changed, 77 insertions(+), 2 deletions(-)
tools/lib/perf/evsel.c +14 -0
index 952f3520d5c2..fc23670231cb 100644
@@ -305,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	if (read_format & PERF_FORMAT_ID)
 		entry += sizeof(u64);
 
+	if (read_format & PERF_FORMAT_LOST)
+		entry += sizeof(u64);
+
 	if (read_format & PERF_FORMAT_GROUP) {
 		nr = evsel->nr_members;
 		size += sizeof(u64);
@@ -314,24 +317,93 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
 	return size;
 }
 
+/* This only reads values for the leader */
+static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
+				  int thread, struct perf_counts_values *count)
+{
+	size_t size = perf_evsel__read_size(evsel);
+	int *fd = FD(evsel, cpu_map_idx, thread);
+	u64 read_format = evsel->attr.read_format;
+	u64 *data;
+	int idx = 1;
+
+	if (fd == NULL || *fd < 0)

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 340

Quoted 10 lines
tools/lib/perf/evsel.c +10 -0
+		return -EINVAL;
+
+	data = calloc(1, size);
+	if (data == NULL)
+		return -ENOMEM;
+
+	if (readn(*fd, data, size) <= 0) {
+		free(data);
+		return -errno;
+	}

could you please put in here some comment that this is intentionaly reading only the leader or better yet rename the function? I was lost before I got to read the changelog ;-)

Quoted 31 lines
tools/lib/perf/evsel.c +31 -0
+
+	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		count->ena = data[idx++];
+	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		count->run = data[idx++];
+
+	/* value is always available */
+	count->val = data[idx++];
+	if (read_format & PERF_FORMAT_ID)
+		count->id = data[idx++];
+	if (read_format & PERF_FORMAT_LOST)
+		count->lost = data[idx++];
+
+	free(data);
+	return 0;
+}
+
+/*
+ * The perf read format is very flexible.  It needs to set the proper
+ * values according to the read format.
+ */
+static void perf_evsel__adjust_values(struct perf_evsel *evsel,
+				      struct perf_counts_values *count)
+{
+	u64 read_format = evsel->attr.read_format;
+
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)) {
+		memmove(&count->values[2], &count->values[1], 24);
+		count->ena = 0;
+	}
+

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 381

Quoted 10 lines
tools/lib/perf/evsel.c +10 -0
+	if (!(read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)) {
+		memmove(&count->values[3], &count->values[2], 16);
+		count->run = 0;
+	}
+
+	if (!(read_format & PERF_FORMAT_ID)) {
+		memmove(&count->values[4], &count->values[3], 8);
+		count->id = 0;
+	}
+}

could we do this the same way we read group counters.. like make read into local buffer and initialize perf_counts_values values based on format, something like:

Quoted 13 lines
        readn(fd, data ...

        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
                count->ena = data[idx++];
        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
                count->run = data[idx++];

        /* value is always available */
        count->val = data[idx++];
        if (read_format & PERF_FORMAT_ID)
                count->id = data[idx++];
        if (read_format & PERF_FORMAT_LOST)
                count->lost = data[idx++];

I see this now, so I'll wait for Namhyung to react to your comments, while keeping it in my local tree just to have it build tested.

- Arnaldo

Quoted 5 lines

and perhaps we should cancel that perf_counts_values's union and keep only 'val/ena/run...' fields?

jirka

Signature · 1 line
- Arnaldo
author maintainer
#5 lore raw

Re: [PATCH 2/4] tools lib perf: Handle read format in perf_evsel__read()

In reply to Jiri Olsa

Hi Jiri,

Quoted 55 lines · Jiri Olsa
Quoted 53 lines · Namhyung Kim

The perf_counts_values should be increased to read the new lost data. Also adjust values after read according the read format.

This supports PERF_FORMAT_GROUP which has a different data format but it's only available for leader events. Currently it doesn't have an API to read sibling (member) events in the group. But users may read the sibling event directly.

Also reading from mmap would be disabled when the read format has ID or LOST bit as it's not exposed via mmap.

---
 tools/lib/perf/evsel.c              | 72 +++++++++++++++++++++++++++++
 tools/lib/perf/include/perf/event.h |  3 +-
 tools/lib/perf/include/perf/evsel.h |  4 +-
 3 files changed, 77 insertions(+), 2 deletions(-)
tools/lib/perf/evsel.c +20 -0
index 952f3520d5c2..fc23670231cb 100644
@@ -305,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
      if (read_format & PERF_FORMAT_ID)
              entry += sizeof(u64);
 
+     if (read_format & PERF_FORMAT_LOST)
+             entry += sizeof(u64);
+
      if (read_format & PERF_FORMAT_GROUP) {
              nr = evsel->nr_members;
              size += sizeof(u64);
@@ -314,24 +317,93 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
      return size;
 }
 
+/* This only reads values for the leader */
+static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
+                               int thread, struct perf_counts_values *count)
+{
+     size_t size = perf_evsel__read_size(evsel);
+     int *fd = FD(evsel, cpu_map_idx, thread);
+     u64 read_format = evsel->attr.read_format;
+     u64 *data;
+     int idx = 1;
+
+     if (fd == NULL || *fd < 0)
+             return -EINVAL;
+
+     data = calloc(1, size);
+     if (data == NULL)
+             return -ENOMEM;
+
Quoted 8 lines

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 340

Quoted 4 lines
tools/lib/perf/evsel.c +4 -0
+     if (readn(*fd, data, size) <= 0) {
+             free(data);
+             return -errno;
+     }

could you please put in here some comment that this is intentionaly reading only the leader or better yet rename the function? I was lost before I got to read the changelog ;-)

Sure, actually it has a comment above the function signature. Maybe we can add an API to read whole group counters but it could be a follow up.

Quoted 48 lines
Quoted 32 lines
tools/lib/perf/evsel.c +31 -0
+
+     if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+             count->ena = data[idx++];
+     if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+             count->run = data[idx++];
+
+     /* value is always available */
+     count->val = data[idx++];
+     if (read_format & PERF_FORMAT_ID)
+             count->id = data[idx++];
+     if (read_format & PERF_FORMAT_LOST)
+             count->lost = data[idx++];
+
+     free(data);
+     return 0;
+}
+
+/*
+ * The perf read format is very flexible.  It needs to set the proper
+ * values according to the read format.
+ */
+static void perf_evsel__adjust_values(struct perf_evsel *evsel,
+                                   struct perf_counts_values *count)
+{
+     u64 read_format = evsel->attr.read_format;
+
+     if (!(read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)) {
+             memmove(&count->values[2], &count->values[1], 24);
+             count->ena = 0;
+     }
+

tools/lib/perf/evsel.c int perf_evsel__read_size(struct perf_evsel *evsel) · line 381

Quoted 10 lines
tools/lib/perf/evsel.c +10 -0
+     if (!(read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)) {
+             memmove(&count->values[3], &count->values[2], 16);
+             count->run = 0;
+     }
+
+     if (!(read_format & PERF_FORMAT_ID)) {
+             memmove(&count->values[4], &count->values[3], 8);
+             count->id = 0;
+     }
+}

could we do this the same way we read group counters.. like make read into local buffer and initialize perf_counts_values values based on format, something like:

Quoted 13 lines
        readn(fd, data ...

        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
                count->ena = data[idx++];
        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
                count->run = data[idx++];

        /* value is always available */
        count->val = data[idx++];
        if (read_format & PERF_FORMAT_ID)
                count->id = data[idx++];
        if (read_format & PERF_FORMAT_LOST)
                count->lost = data[idx++];

Sure.

Quoted 4 lines

and perhaps we should cancel that perf_counts_values's union and keep only 'val/ena/run...' fields?

I'd like to have them and use it for reading the lost count soon.

Thanks,
Namhyung

author maintainer
#6 lore raw

PATCH 3/4 tools lib perf: Add a test case for read formats

In reply to Namhyung Kim

It checks a various combination of the read format settings and verify it return the value in a proper position. The test uses task-clock software events to guarantee it's always active and sets enabled/running time.

---
 tools/lib/perf/tests/test-evsel.c | 161 ++++++++++++++++++++++++++++++
 1 file changed, 161 insertions(+)
tools/lib/perf/tests/test-evsel.c +161 -0
index 89be89afb24d..a11fc51bfb68 100644
@@ -1,10 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <stdarg.h>
 #include <stdio.h>
+#include <string.h>
 #include <linux/perf_event.h>
+#include <linux/kernel.h>
 #include <perf/cpumap.h>
 #include <perf/threadmap.h>
 #include <perf/evsel.h>
+#include <internal/evsel.h>
 #include <internal/tests.h>
 #include "tests.h"
 
@@ -189,6 +192,163 @@ static int test_stat_user_read(int event)
 	return 0;
 }
 
+static int test_stat_read_format_single(struct perf_event_attr *attr, struct perf_thread_map *threads)
+{
+	struct perf_evsel *evsel;
+	struct perf_counts_values counts;
+	volatile int count = 0x100000;
+	int err;
+
+	evsel = perf_evsel__new(attr);
+	__T("failed to create evsel", evsel);
+
+	/* skip old kernels that don't support the format */
+	err = perf_evsel__open(evsel, NULL, threads);
+	if (err < 0)
+		return 0;
+
+	while (count--) ;
+
+	memset(&counts, -1, sizeof(counts));
+	perf_evsel__read(evsel, 0, 0, &counts);
+
+	__T("failed to read value", counts.val);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		__T("failed to read TOTAL_TIME_ENABLED", counts.ena);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		__T("failed to read TOTAL_TIME_RUNNING", counts.run);
+	if (attr->read_format & PERF_FORMAT_ID)
+		__T("failed to read ID", counts.id);
+	if (attr->read_format & PERF_FORMAT_LOST)
+		__T("failed to read LOST", counts.lost == 0);
+
+	perf_evsel__close(evsel);
+	perf_evsel__delete(evsel);
+	return 0;
+}
+
+static int test_stat_read_format_group(struct perf_event_attr *attr, struct perf_thread_map *threads)
+{
+	struct perf_evsel *leader, *member;
+	struct perf_counts_values counts;
+	volatile int count = 0x100000;
+	int err;
+
+	attr->read_format |= PERF_FORMAT_GROUP;
+	leader = perf_evsel__new(attr);
+	__T("failed to create leader", leader);
+
+	attr->read_format &= ~PERF_FORMAT_GROUP;
+	member = perf_evsel__new(attr);
+	__T("failed to create member", member);
+
+	member->leader = leader;
+	leader->nr_members = 2;
+
+	/* skip old kernels that don't support the format */
+	err = perf_evsel__open(leader, NULL, threads);
+	if (err < 0)
+		return 0;
+	err = perf_evsel__open(member, NULL, threads);
+	if (err < 0)
+		return 0;
+
+	while (count--) ;
+
+	memset(&counts, -1, sizeof(counts));
+	perf_evsel__read(leader, 0, 0, &counts);
+
+	__T("failed to read leader value", counts.val);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		__T("failed to read leader TOTAL_TIME_ENABLED", counts.ena);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		__T("failed to read leader TOTAL_TIME_RUNNING", counts.run);
+	if (attr->read_format & PERF_FORMAT_ID)
+		__T("failed to read leader ID", counts.id);
+	if (attr->read_format & PERF_FORMAT_LOST)
+		__T("failed to read leader LOST", counts.lost == 0);
+
+	memset(&counts, -1, sizeof(counts));
+	perf_evsel__read(member, 0, 0, &counts);
+
+	__T("failed to read member value", counts.val);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+		__T("failed to read member TOTAL_TIME_ENABLED", counts.ena);
+	if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+		__T("failed to read member TOTAL_TIME_RUNNING", counts.run);
+	if (attr->read_format & PERF_FORMAT_ID)
+		__T("failed to read member ID", counts.id);
+	if (attr->read_format & PERF_FORMAT_LOST)
+		__T("failed to read member LOST", counts.lost == 0);
+
+	perf_evsel__close(member);
+	perf_evsel__close(leader);
+	perf_evsel__delete(member);
+	perf_evsel__delete(leader);
+	return 0;
+}
+
+static int test_stat_read_format(void)
+{
+	struct perf_thread_map *threads;
+	struct perf_event_attr attr = {
+		.type	= PERF_TYPE_SOFTWARE,
+		.config	= PERF_COUNT_SW_TASK_CLOCK,
+	};
+	int err, i;
+
+#define FMT(_fmt)  PERF_FORMAT_ ## _fmt
+#define FMT_TIME  (FMT(TOTAL_TIME_ENABLED) | FMT(TOTAL_TIME_RUNNING))
+
+	uint64_t test_formats [] = {
+		0,
+		FMT_TIME,
+		FMT(ID),
+		FMT(LOST),
+		FMT_TIME | FMT(ID),
+		FMT_TIME | FMT(LOST),
+		FMT_TIME | FMT(ID) | FMT(LOST),
+		FMT(ID) | FMT(LOST),
+	};
+
+#undef FMT
+#undef FMT_TIME
+
+	threads = perf_thread_map__new_dummy();
+	__T("failed to create threads", threads);
+
+	perf_thread_map__set_pid(threads, 0, 0);
+
+	for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) {
+		attr.read_format = test_formats[i];
+		__T_VERBOSE("testing single read with read_format: %lx\n",
+			    (unsigned long)test_formats[i]);
+
+		err = test_stat_read_format_single(&attr, threads);
+		__T("failed to read single format", err == 0);
+	}
+
+	perf_thread_map__put(threads);
+
+	threads = perf_thread_map__new_array(2, NULL);
+	__T("failed to create threads", threads);
+
+	perf_thread_map__set_pid(threads, 0, 0);
+	perf_thread_map__set_pid(threads, 1, 0);
+
+	for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) {
+		attr.read_format = test_formats[i];
+		__T_VERBOSE("testing group read with read_format: %lx\n",
+			    (unsigned long)test_formats[i]);
+
+		err = test_stat_read_format_group(&attr, threads);
+		__T("failed to read group format", err == 0);
+	}
+
+	perf_thread_map__put(threads);
+	return 0;
+}
+
 int test_evsel(int argc, char **argv)
 {
 	__T_START;
@@ -200,6 +360,7 @@ int test_evsel(int argc, char **argv)
 	test_stat_thread_enable();
 	test_stat_user_read(PERF_COUNT_HW_INSTRUCTIONS);
 	test_stat_user_read(PERF_COUNT_HW_CPU_CYCLES);
+	test_stat_read_format();
 
 	__T_END;
 	return tests_failed == 0 ? 0 : -1;
Signature · 1 line
2.37.1.595.g718a3a8f04-goog
author maintainer
#7 lore raw

PATCH 4/4 perf tools: Support reading PERF_FORMAT_LOST

In reply to Namhyung Kim

The recent kernel added lost count can be read from either read(2) or ring buffer data with PERF_SAMPLE_READ. As it's a variable length data we need to access it according to the format info.

But for perf tools use cases, PERF_FORMAT_ID is always set. So we can only check PERF_FORMAT_LOST bit to determine the data format.

Add sample_read_value_size() and next_sample_read_value() helpers to make it a bit easier to access. Use them in all places where it reads the struct sample_read_value.

---
 tools/perf/tests/sample-parsing.c             | 14 +++++---
 tools/perf/util/event.h                       | 18 +++++++++-
 tools/perf/util/evsel.c                       | 33 +++++++++++++-----
 .../scripting-engines/trace-event-python.c    | 16 ++++++---
 tools/perf/util/session.c                     | 32 +++++++++++------
 tools/perf/util/synthetic-events.c            | 34 +++++++++++++++----
 6 files changed, 112 insertions(+), 35 deletions(-)
tools/perf/tests/sample-parsing.c +10 -4
index 07f2411b0ad4..20930dd48ee0 100644
@@ -86,10 +86,15 @@ static bool samples_same(const struct perf_sample *s1,
 			COMP(read.time_running);
 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
 		if (read_format & PERF_FORMAT_GROUP) {
-			for (i = 0; i < s1->read.group.nr; i++)
-				MCOMP(read.group.values[i]);
+			for (i = 0; i < s1->read.group.nr; i++) {
+				/* FIXME: check values without LOST */
+				if (read_format & PERF_FORMAT_LOST)
+					MCOMP(read.group.values[i]);
+			}
 		} else {
 			COMP(read.one.id);
+			if (read_format & PERF_FORMAT_LOST)
+				COMP(read.one.lost);
 		}
 	}
 
@@ -263,7 +268,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 			.data	= (void *)aux_data,
 		},
 	};
-	struct sample_read_value values[] = {{1, 5}, {9, 3}, {2, 7}, {6, 4},};
+	struct sample_read_value values[] = {{1, 5, 0}, {9, 3, 0}, {2, 7, 0}, {6, 4, 1},};
 	struct perf_sample sample_out, sample_out_endian;
 	size_t i, sz, bufsz;
 	int err, ret = -1;
@@ -286,6 +291,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 	} else {
 		sample.read.one.value = 0x08789faeb786aa87ULL;
 		sample.read.one.id    = 99;
+		sample.read.one.lost  = 1;
 	}
 
 	sz = perf_event__sample_event_size(&sample, sample_type, read_format);
@@ -370,7 +376,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
  */
 static int test__sample_parsing(struct test_suite *test __maybe_unused, int subtest __maybe_unused)
 {
-	const u64 rf[] = {4, 5, 6, 7, 12, 13, 14, 15};
+	const u64 rf[] = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 28, 29, 30, 31};
 	u64 sample_type;
 	u64 sample_regs;
 	size_t i;
tools/perf/util/event.h +17 -1
index a7b0931d5137..7753368d70d6 100644
@@ -65,7 +65,8 @@ struct stack_dump {
 
 struct sample_read_value {
 	u64 value;
-	u64 id;
+	u64 id;   /* only if PERF_FORMAT_ID */
+	u64 lost; /* only if PERF_FORMAT_LOST */
 };
 
 struct sample_read {
@@ -80,6 +81,21 @@ struct sample_read {
 	};
 };
 
+static inline size_t sample_read_value_size(u64 read_format)
+{
+	/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
+	if (read_format & PERF_FORMAT_LOST)
+		return sizeof(struct sample_read_value);
+	else
+		return offsetof(struct sample_read_value, lost);
+}
+
+static inline struct sample_read_value *
+next_sample_read_value(struct sample_read_value *v, u64 read_format)
+{
+	return (void *)v + sample_read_value_size(read_format);
+}
+
 struct ip_callchain {
 	u64 nr;
 	u64 ips[];
tools/perf/util/evsel.c +24 -9
index 4852089e1d79..781367ccdb84 100644
@@ -1541,7 +1541,7 @@ static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread)
 }
 
 static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread,
-			     u64 val, u64 ena, u64 run)
+			     u64 val, u64 ena, u64 run, u64 lost)
 {
 	struct perf_counts_values *count;
 
@@ -1550,6 +1550,7 @@ static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread,
 	count->val    = val;
 	count->ena    = ena;
 	count->run    = run;
+	count->lost   = lost;
 
 	perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true);
 }
@@ -1558,7 +1559,7 @@ static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int
 {
 	u64 read_format = leader->core.attr.read_format;
 	struct sample_read_value *v;
-	u64 nr, ena = 0, run = 0, i;
+	u64 nr, ena = 0, run = 0, lost = 0, i;
 
 	nr = *data++;
 
@@ -1573,16 +1574,25 @@ static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int
 
 	v = (struct sample_read_value *) data;
 
-	evsel__set_count(leader, cpu_map_idx, thread, v[0].value, ena, run);
+	if (read_format & PERF_FORMAT_LOST)
+		lost = v->lost;
+
+	evsel__set_count(leader, cpu_map_idx, thread, v[0].value, ena, run, lost);
+
+	v = next_sample_read_value(v, read_format);
 
 	for (i = 1; i < nr; i++) {
 		struct evsel *counter;
 
-		counter = evlist__id2evsel(leader->evlist, v[i].id);
+		counter = evlist__id2evsel(leader->evlist, v->id);
 		if (!counter)
 			return -EINVAL;
 
-		evsel__set_count(counter, cpu_map_idx, thread, v[i].value, ena, run);
+		if (read_format & PERF_FORMAT_LOST)
+			lost = v->lost;
+
+		evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost);
+		v = next_sample_read_value(v, read_format);
 	}
 
 	return 0;
@@ -2475,16 +2485,21 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
 
 			if (data->read.group.nr > max_group_nr)
 				return -EFAULT;
-			sz = data->read.group.nr *
-			     sizeof(struct sample_read_value);
+
+			sz = data->read.group.nr * sample_read_value_size(read_format);
 			OVERFLOW_CHECK(array, sz, max_size);
-			data->read.group.values =
-					(struct sample_read_value *)array;
+			data->read.group.values = (void *)array;
 			array = (void *)array + sz;
 		} else {
 			OVERFLOW_CHECK_u64(array);
 			data->read.one.id = *array;
 			array++;
+
+			if (read_format & PERF_FORMAT_LOST) {
+				OVERFLOW_CHECK_u64(array);
+				data->read.one.lost = *array;
+				array++;
+			}
 		}
 	}
 
tools/perf/util/scripting-engines/trace-event-python.c +12 -4
index 9ef2406e0ede..7e15e64f311b 100644
@@ -642,15 +642,19 @@ static PyObject *python_process_brstacksym(struct perf_sample *sample,
 	return pylist;
 }
 
-static PyObject *get_sample_value_as_tuple(struct sample_read_value *value)
+static PyObject *get_sample_value_as_tuple(struct sample_read_value *value,
+					   u64 read_format)
 {
 	PyObject *t;
 
-	t = PyTuple_New(2);
+	t = PyTuple_New(3);
 	if (!t)
 		Py_FatalError("couldn't create Python tuple");
 	PyTuple_SetItem(t, 0, PyLong_FromUnsignedLongLong(value->id));
 	PyTuple_SetItem(t, 1, PyLong_FromUnsignedLongLong(value->value));
+	if (read_format & PERF_FORMAT_LOST)
+		PyTuple_SetItem(t, 2, PyLong_FromUnsignedLongLong(value->lost));
+
 	return t;
 }
 
@@ -681,12 +685,16 @@ static void set_sample_read_in_dict(PyObject *dict_sample,
 		Py_FatalError("couldn't create Python list");
 
 	if (read_format & PERF_FORMAT_GROUP) {
+		struct sample_read_value *v = sample->read.group.values;
+
 		for (i = 0; i < sample->read.group.nr; i++) {
-			PyObject *t = get_sample_value_as_tuple(&sample->read.group.values[i]);
+			PyObject *t = get_sample_value_as_tuple(v, read_format);
 			PyList_SET_ITEM(values, i, t);
+			v = next_sample_read_value(v, read_format);
 		}
 	} else {
-		PyObject *t = get_sample_value_as_tuple(&sample->read.one);
+		PyObject *t = get_sample_value_as_tuple(&sample->read.one,
+							read_format);
 		PyList_SET_ITEM(values, 0, t);
 	}
 	pydict_set_item_string_decref(dict_sample, "values", values);
tools/perf/util/session.c +22 -10
index 98e16659a149..dd818764bd4d 100644
@@ -1284,20 +1284,26 @@ static void sample_read__printf(struct perf_sample *sample, u64 read_format)
 
 	if (read_format & PERF_FORMAT_GROUP) {
 		u64 i;
+		struct sample_read_value *value = sample->read.group.values;
 
 		printf(".... group nr %" PRIu64 "\n", sample->read.group.nr);
 
 		for (i = 0; i < sample->read.group.nr; i++) {
-			struct sample_read_value *value;
-
-			value = &sample->read.group.values[i];
 			printf("..... id %016" PRIx64
-			       ", value %016" PRIx64 "\n",
+			       ", value %016" PRIx64,
 			       value->id, value->value);
+			if (read_format & PERF_FORMAT_LOST)
+				printf(", lost %" PRIu64, value->lost);
+			printf("\n");
+			value = next_sample_read_value(value, read_format);
 		}
-	} else
-		printf("..... id %016" PRIx64 ", value %016" PRIx64 "\n",
+	} else {
+		printf("..... id %016" PRIx64 ", value %016" PRIx64,
 			sample->read.one.id, sample->read.one.value);
+		if (read_format & PERF_FORMAT_LOST)
+			printf(", lost %" PRIu64, sample->read.one.lost);
+		printf("\n");
+	}
 }
 
 static void dump_event(struct evlist *evlist, union perf_event *event,
@@ -1411,6 +1417,9 @@ static void dump_read(struct evsel *evsel, union perf_event *event)
 
 	if (read_format & PERF_FORMAT_ID)
 		printf("... id           : %" PRI_lu64 "\n", read_event->id);
+
+	if (read_format & PERF_FORMAT_LOST)
+		printf("... lost         : %" PRI_lu64 "\n", read_event->lost);
 }
 
 static struct machine *machines__find_for_cpumode(struct machines *machines,
@@ -1479,17 +1488,20 @@ static int deliver_sample_group(struct evlist *evlist,
 				struct perf_tool *tool,
 				union  perf_event *event,
 				struct perf_sample *sample,
-				struct machine *machine)
+				struct machine *machine,
+				u64 read_format)
 {
 	int ret = -EINVAL;
 	u64 i;
+	struct sample_read_value *v = sample->read.group.values;
 
 	for (i = 0; i < sample->read.group.nr; i++) {
-		ret = deliver_sample_value(evlist, tool, event, sample,
-					   &sample->read.group.values[i],
+		ret = deliver_sample_value(evlist, tool, event, sample, v,
 					   machine);
 		if (ret)
 			break;
+
+		v = next_sample_read_value(v, read_format);
 	}
 
 	return ret;
@@ -1510,7 +1522,7 @@ static int evlist__deliver_sample(struct evlist *evlist, struct perf_tool *tool,
 	/* For PERF_SAMPLE_READ we have either single or group mode. */
 	if (read_format & PERF_FORMAT_GROUP)
 		return deliver_sample_group(evlist, tool, event, sample,
-					    machine);
+					    machine, read_format);
 	else
 		return deliver_sample_value(evlist, tool, event, sample,
 					    &sample->read.one, machine);
tools/perf/util/synthetic-events.c +27 -7
index 2ae59c03ae77..9b8dc7633a02 100644
@@ -1432,11 +1432,12 @@ size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type,
 			result += sizeof(u64);
 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
 		if (read_format & PERF_FORMAT_GROUP) {
-			sz = sample->read.group.nr *
-			     sizeof(struct sample_read_value);
-			result += sz;
+			sz = sample_read_value_size(read_format);
+			result += sz * sample->read.group.nr;
 		} else {
 			result += sizeof(u64);
+			if (read_format & PERF_FORMAT_LOST)
+				result += sizeof(u64);
 		}
 	}
 
@@ -1521,6 +1522,22 @@ void __weak arch_perf_synthesize_sample_weight(const struct perf_sample *data,
 	*array = data->weight;
 }
 
+static __u64 *copy_read_group_values(__u64 *array, __u64 read_format,
+				     const struct perf_sample *sample)
+{
+	size_t i, sz;
+	void *data = sample->read.group.values;
+
+	sz = sample_read_value_size(read_format);
+	for (i = 0; i < sample->read.group.nr; i++) {
+		memcpy(array, data, sz);
+
+		data += sz;
+		array = (void *)array + sz;
+	}
+	return array;
+}
+
 int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format,
 				  const struct perf_sample *sample)
 {
@@ -1602,13 +1619,16 @@ int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_fo
 
 		/* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
 		if (read_format & PERF_FORMAT_GROUP) {
-			sz = sample->read.group.nr *
-			     sizeof(struct sample_read_value);
-			memcpy(array, sample->read.group.values, sz);
-			array = (void *)array + sz;
+			array = copy_read_group_values(array, read_format,
+						       sample);
 		} else {
 			*array = sample->read.one.id;
 			array++;
+
+			if (read_format & PERF_FORMAT_LOST) {
+				*array = sample->read.one.lost;
+				array++;
+			}
 		}
 	}
 
Signature · 1 line
2.37.1.595.g718a3a8f04-goog
#8 lore raw
In reply to Namhyung Kim

Em Mon, Aug 15, 2022 at 12:01:02PM -0700, Namhyung Kim escreveu:

Quoted 3 lines

Hello,

The kernel v6.0 added PERF_FORMAT_LOST which can read a number of lost

I've tentantively added this to perf/urgent, as this matches a feature introduced in this merge window, we'll see how it goes, hopefully Linus accepts this at this point.

- Arnaldo

Quoted 34 lines

samples for the given event. As it can change the output format of read(2) and perf sample data, it needs to access them carefully.

You can get the code from 'perf/read-lost-v1' brach on

git://git.kernel.org/pub/scm/linux/kernel/git/namhyung/linux-perf.git

Thanks,
Namhyung

Namhyung Kim (4):
tools headers UAPI: Sync linux/perf_event.h with the kernel sources tools lib perf: Handle read format in perf_evsel__read() tools lib perf: Add a test case for read formats
perf tools: Support reading PERF_FORMAT_LOST

 tools/include/uapi/linux/perf_event.h         |   5 +-
 tools/lib/perf/evsel.c                        |  72 ++++++++
 tools/lib/perf/include/perf/event.h           |   3 +-
 tools/lib/perf/include/perf/evsel.h           |   4 +-
 tools/lib/perf/tests/test-evsel.c             | 161 ++++++++++++++++++
 tools/perf/tests/sample-parsing.c             |  14 +-
 tools/perf/util/event.h                       |  18 +-
 tools/perf/util/evsel.c                       |  33 +++-
 .../scripting-engines/trace-event-python.c    |  16 +-
 tools/perf/util/session.c                     |  32 ++--
 tools/perf/util/synthetic-events.c            |  34 +++-
 11 files changed, 354 insertions(+), 38 deletions(-)

base-commit: 568035b01cfb107af8d2e4bd2fb9aea22cf5b868

Signature · 1 line
2.37.1.595.g718a3a8f04-goog
Signature · 1 line
- Arnaldo