Skip to contentSkip to conversation

perf expr: Test parsing of floating point numbers

PATCH Superseded Ian Rogers 3 participants 11 messages May 13, 2020 – May 15, 2020 perf events

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Ian Rogers

author reviewer
#0 lore raw

PATCH 1/2 perf expr: Test parsing of floating point numbers

Add test for fix in:
commit 5741da3dee4c ("perf expr: Parse numbers as doubles")

---
 tools/perf/tests/expr.c | 1 +
 1 file changed, 1 insertion(+)
tools/perf/tests/expr.c +1 -0
index f9e8e5628836..3f742612776a 100644
@@ -39,6 +39,7 @@ int test__expr(struct test *t __maybe_unused, int subtest __maybe_unused)
 	ret |= test(&ctx, "min(1,2) + 1", 2);
 	ret |= test(&ctx, "max(1,2) + 1", 3);
 	ret |= test(&ctx, "1+1 if 3*4 else 0", 2);
+	ret |= test(&ctx, "1.1 + 2.1", 3.2);
 
 	if (ret)
 		return ret;
Signature · 1 line
2.26.2.645.ge9eca65c58-goog
author reviewer
#1 lore raw

PATCH 2/2 perf test: Improve pmu event metric testing

Break pmu-events test into 2 and add a test to verify that all pmu metric expressions simply parse. Try to parse all metric ids/events, failing if metrics for the current architecture fail to parse.

Tested on power9, skylakex, haswell, broadwell, westmere, sandybridge and ivybridge with the patch set in place.
May fail on other architectures if metrics are invalid. In particular s390 is untested, but its expressions are trivial. The event encodings could be wrong. The other untested architectures with expressions are power8, cascadelakex, tremontx, skylake, jaketown, ivytown and variants of haswell and broadwell. For these the expression encoding is valid but the event encodings may not be.

---
 tools/perf/tests/builtin-test.c |   5 +
 tools/perf/tests/pmu-events.c   | 158 ++++++++++++++++++++++++++++++--
 tools/perf/tests/tests.h        |   2 +
 3 files changed, 159 insertions(+), 6 deletions(-)
tools/perf/tests/builtin-test.c +5 -0
index 3471ec52ea11..8147c17c71ab 100644
@@ -75,6 +75,11 @@ static struct test generic_tests[] = {
 	{
 		.desc = "PMU events",
 		.func = test__pmu_events,
+		.subtest = {
+			.get_nr		= test__pmu_events_subtest_get_nr,
+			.get_desc	= test__pmu_events_subtest_get_desc,
+		},
+
 	},
 	{
 		.desc = "DSO data read",
tools/perf/tests/pmu-events.c +152 -6

182 lines not on this page · raw message

tools/perf/tests/tests.h +2 -0
index d6d4ac34eeb7..8e316c30ed3c 100644
@@ -50,6 +50,8 @@ int test__perf_evsel__tp_sched_test(struct test *test, int subtest);
 int test__syscall_openat_tp_fields(struct test *test, int subtest);
 int test__pmu(struct test *test, int subtest);
 int test__pmu_events(struct test *test, int subtest);
+const char *test__pmu_events_subtest_get_desc(int subtest);
+int test__pmu_events_subtest_get_nr(void);
 int test__attr(struct test *test, int subtest);
 int test__dso_data(struct test *test, int subtest);
 int test__dso_data_cache(struct test *test, int subtest);
Signature · 1 line
2.26.2.645.ge9eca65c58-goog
author reviewer
#2 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 5 lines · Ian Rogers

Break pmu-events test into 2 and add a test to verify that all pmu metric expressions simply parse. Try to parse all metric ids/events, failing if metrics for the current architecture fail to parse.

Quoted 8 lines

Tested on power9, skylakex, haswell, broadwell, westmere, sandybridge and ivybridge with the patch set in place.
May fail on other architectures if metrics are invalid. In particular s390 is untested, but its expressions are trivial. The event encodings could be wrong. The other untested architectures with expressions are power8, cascadelakex, tremontx, skylake, jaketown, ivytown and variants of haswell and broadwell. For these the expression encoding is valid but the event encodings may not be.

Sorry, this commit message needed updating to reflect that event parsing errors are no longer fatal. Fixed in v2:
https://lore.kernel.org/lkml/[email protected]/T/#t

Thanks,
Ian

Quoted 229 lines
---
 tools/perf/tests/builtin-test.c |   5 +
 tools/perf/tests/pmu-events.c   | 158 ++++++++++++++++++++++++++++++--
 tools/perf/tests/tests.h        |   2 +
 3 files changed, 159 insertions(+), 6 deletions(-)
tools/perf/tests/builtin-test.c +5 -0
index 3471ec52ea11..8147c17c71ab 100644
@@ -75,6 +75,11 @@ static struct test generic_tests[] = {
        {
                .desc = "PMU events",
                .func = test__pmu_events,
+               .subtest = {
+                       .get_nr         = test__pmu_events_subtest_get_nr,
+                       .get_desc       = test__pmu_events_subtest_get_desc,
+               },
+
        },
        {
                .desc = "DSO data read",
tools/perf/tests/pmu-events.c +152 -6
index d64261da8bf7..c18b9ce8cace 100644
@@ -8,6 +8,10 @@
 #include <linux/zalloc.h>
 #include "debug.h"
 #include "../pmu-events/pmu-events.h"
+#include "util/evlist.h"
+#include "util/expr.h"
+#include "util/parse-events.h"
+#include <ctype.h>
 
 struct perf_pmu_test_event {
        struct pmu_event event;
@@ -144,7 +148,7 @@ static struct pmu_events_map *__test_pmu_get_events_map(void)
 }
 
 /* Verify generated events from pmu-events.c is as expected */
-static int __test_pmu_event_table(void)
+static int test_pmu_event_table(void)
 {
        struct pmu_events_map *map = __test_pmu_get_events_map();
        struct pmu_event *table;
@@ -347,14 +351,11 @@ static int __test__pmu_event_aliases(char *pmu_name, int *count)
        return res;
 }
 
-int test__pmu_events(struct test *test __maybe_unused,
-                    int subtest __maybe_unused)
+
+static int test_aliases(void)
 {
        struct perf_pmu *pmu = NULL;
 
-       if (__test_pmu_event_table())
-               return -1;
-
        while ((pmu = perf_pmu__scan(pmu)) != NULL) {
                int count = 0;
 
@@ -377,3 +378,148 @@ int test__pmu_events(struct test *test __maybe_unused,
 
        return 0;
 }
+
+static bool is_number(const char *str)
+{
+       size_t i;
+
+       for (i = 0; i < strlen(str); i++) {
+               if (!isdigit(str[i]) && str[i] != '.')
+                       return false;
+       }
+       return true;
+}
+
+static int check_parse_id(const char *id, bool same_cpu, struct pmu_event *pe)
+{
+       struct parse_events_error error;
+       struct evlist *evlist;
+       int ret;
+
+       /* Numbers are always valid. */
+       if (is_number(id))
+               return 0;
+
+       evlist = evlist__new();
+       memset(&error, 0, sizeof(error));
+       ret = parse_events(evlist, id, &error);
+       if (ret && same_cpu) {
+               fprintf(stderr,
+                       "\nWARNING: Parse event failed metric '%s' id '%s' expr '%s'\n",
+                       pe->metric_name, id, pe->metric_expr);
+               fprintf(stderr, "Error string '%s' help '%s'\n",
+                       error.str, error.help);
+       } else if (ret) {
+               pr_debug3("Parse event failed, but for an event that may not be supported by this CPU.\nid '%s' metric '%s' expr '%s'\n",
+                       id, pe->metric_name, pe->metric_expr);
+       }
+       evlist__delete(evlist);
+       free(error.str);
+       free(error.help);
+       free(error.first_str);
+       free(error.first_help);
+       /* TODO: too many metrics are broken to fail on this test currently. */
+       return 0;
+}
+
+static int test_parsing(void)
+{
+       struct pmu_events_map *cpus_map = perf_pmu__find_map(NULL);
+       struct pmu_events_map *map;
+       struct pmu_event *pe;
+       int i, j, k;
+       const char **ids;
+       int idnum;
+       int ret = 0;
+       struct expr_parse_ctx ctx;
+       double result;
+
+       i = 0;
+       for (;;) {
+               map = &pmu_events_map[i++];
+               if (!map->table) {
+                       map = NULL;
+                       break;
+               }
+               j = 0;
+               for (;;) {
+                       pe = &map->table[j++];
+                       if (!pe->name && !pe->metric_group && !pe->metric_name)
+                               break;
+                       if (!pe->metric_expr)
+                               continue;
+                       if (expr__find_other(pe->metric_expr, NULL,
+                                               &ids, &idnum, 0) < 0) {
+                               pr_debug("Parse other failed for map %s %s %s\n",
+                                       map->cpuid, map->version, map->type);
+                               pr_debug("On metric %s\n", pe->metric_name);
+                               pr_debug("On expression %s\n", pe->metric_expr);
+                               ret++;
+                               continue;
+                       }
+                       expr__ctx_init(&ctx);
+
+                       /*
+                        * Add all ids with a made up value. The value may
+                        * trigger divide by zero when subtracted and so try to
+                        * make them unique.
+                        */
+                       for (k = 0; k < idnum; k++)
+                               expr__add_id(&ctx, ids[k], k + 1);
+
+                       for (k = 0; k < idnum; k++) {
+                               if (check_parse_id(ids[k], map == cpus_map, pe))
+                                       ret++;
+                       }
+
+                       if (expr__parse(&result, &ctx, pe->metric_expr, 0)) {
+                               pr_debug("Parse failed for map %s %s %s\n",
+                                       map->cpuid, map->version, map->type);
+                               pr_debug("On metric %s\n", pe->metric_name);
+                               pr_debug("On expression %s\n", pe->metric_expr);
+                               ret++;
+                       }
+                       for (k = 0; k < idnum; k++)
+                               zfree(&ids[k]);
+                       free(ids);
+               }
+       }
+       return ret;
+}
+
+static const struct {
+       int (*func)(void);
+       const char *desc;
+} pmu_events_testcase_table[] = {
+       {
+               .func = test_pmu_event_table,
+               .desc = "PMU event table sanity",
+       },
+       {
+               .func = test_aliases,
+               .desc = "PMU event map aliases",
+       },
+       {
+               .func = test_parsing,
+               .desc = "Parsing of PMU event table metrics",
+       },
+};
+
+const char *test__pmu_events_subtest_get_desc(int i)
+{
+       if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+               return NULL;
+       return pmu_events_testcase_table[i].desc;
+}
+
+int test__pmu_events_subtest_get_nr(void)
+{
+       return (int)ARRAY_SIZE(pmu_events_testcase_table);
+}
+
+int test__pmu_events(struct test *test __maybe_unused, int i)
+{
+       if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+               return TEST_FAIL;
+       return pmu_events_testcase_table[i].func();
+}
tools/perf/tests/tests.h +2 -1
index d6d4ac34eeb7..8e316c30ed3c 100644
@@ -50,6 +50,8 @@ int test__perf_evsel__tp_sched_test(struct test *test, int subtest);
 int test__syscall_openat_tp_fields(struct test *test, int subtest);
 int test__pmu(struct test *test, int subtest);
 int test__pmu_events(struct test *test, int subtest);
+const char *test__pmu_events_subtest_get_desc(int subtest);
+int test__pmu_events_subtest_get_nr(void);
 int test__attr(struct test *test, int subtest);
 int test__dso_data(struct test *test, int subtest);
 int test__dso_data_cache(struct test *test, int subtest);
--

2.26.2.645.ge9eca65c58-goog

#3 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

In reply to Ian Rogers
Quoted 4 lines · Ian Rogers

Break pmu-events test into 2 and add a test to verify that all pmu metric expressions simply parse. Try to parse all metric ids/events, failing if metrics for the current architecture fail to parse.

Quoted 9 lines

Tested on power9, skylakex, haswell, broadwell, westmere, sandybridge and ivybridge with the patch set in place.
May fail on other architectures if metrics are invalid. In particular s390 is untested, but its expressions are trivial. The event encodings could be wrong. The other untested architectures with expressions are power8, cascadelakex, tremontx, skylake, jaketown, ivytown and variants of haswell and broadwell. For these the expression encoding is valid but the event encodings may not be.

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

Quoted 69 lines
---
  tools/perf/tests/builtin-test.c |   5 +
  tools/perf/tests/pmu-events.c   | 158 ++++++++++++++++++++++++++++++--
  tools/perf/tests/tests.h        |   2 +
  3 files changed, 159 insertions(+), 6 deletions(-)
tools/perf/tests/builtin-test.c +5 -0
index 3471ec52ea11..8147c17c71ab 100644
@@ -75,6 +75,11 @@ static struct test generic_tests[] = {
  	{
  		.desc = "PMU events",
  		.func = test__pmu_events,
+		.subtest = {
+			.get_nr		= test__pmu_events_subtest_get_nr,
+			.get_desc	= test__pmu_events_subtest_get_desc,
+		},
+
  	},
  	{
  		.desc = "DSO data read",
tools/perf/tests/pmu-events.c +8 -6
index d64261da8bf7..c18b9ce8cace 100644
@@ -8,6 +8,10 @@
  #include <linux/zalloc.h>
  #include "debug.h"
  #include "../pmu-events/pmu-events.h"
+#include "util/evlist.h"
+#include "util/expr.h"
+#include "util/parse-events.h"
+#include <ctype.h>
 
  struct perf_pmu_test_event {
  	struct pmu_event event;
@@ -144,7 +148,7 @@ static struct pmu_events_map *__test_pmu_get_events_map(void)
  }
 
  /* Verify generated events from pmu-events.c is as expected */
-static int __test_pmu_event_table(void)
+static int test_pmu_event_table(void)
  {
  	struct pmu_events_map *map = __test_pmu_get_events_map();
  	struct pmu_event *table;
@@ -347,14 +351,11 @@ static int __test__pmu_event_aliases(char *pmu_name, int *count)
  	return res;
  }
 
-int test__pmu_events(struct test *test __maybe_unused,
-		     int subtest __maybe_unused)
+
+static int test_aliases(void)
  {
  	struct perf_pmu *pmu = NULL;
 
-	if (__test_pmu_event_table())
-		return -1;
-
  	while ((pmu = perf_pmu__scan(pmu)) != NULL) {
  		int count = 0;
 
@@ -377,3 +378,148 @@ int test__pmu_events(struct test *test __maybe_unused,
 
  	return 0;
  }
+

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 391

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+static bool is_number(const char *str)
+{
+	size_t i;
+
+	for (i = 0; i < strlen(str); i++) {
+		if (!isdigit(str[i]) && str[i] != '.')
+			return false;
+	}
+	return true;
+}

if there is no helper already for this, is this really the best place to put it? Maybe others will need it in future.

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+
+static int check_parse_id(const char *id, bool same_cpu, struct pmu_event *pe)
+{
+	struct parse_events_error error;
+	struct evlist *evlist;
+	int ret;
+
+	/* Numbers are always valid. */
+	if (is_number(id))
+		return 0;

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 411

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+
+	evlist = evlist__new();
+	memset(&error, 0, sizeof(error));
+	ret = parse_events(evlist, id, &error);
+	if (ret && same_cpu) {
+		fprintf(stderr,
+			"\nWARNING: Parse event failed metric '%s' id '%s' expr '%s'\n",
+			pe->metric_name, id, pe->metric_expr);
+		fprintf(stderr, "Error string '%s' help '%s'\n",
+			error.str, error.help);

do you really need to spill lines like this?

Quoted 29 lines
tools/perf/tests/pmu-events.c +29 -0
+	} else if (ret) {
+		pr_debug3("Parse event failed, but for an event that may not be supported by this CPU.\nid '%s' metric '%s' expr '%s'\n",
+			id, pe->metric_name, pe->metric_expr);
+	}
+	evlist__delete(evlist);
+	free(error.str);
+	free(error.help);
+	free(error.first_str);
+	free(error.first_help);
+	/* TODO: too many metrics are broken to fail on this test currently. */
+	return 0;
+}
+
+static int test_parsing(void)
+{
+	struct pmu_events_map *cpus_map = perf_pmu__find_map(NULL);
+	struct pmu_events_map *map;
+	struct pmu_event *pe;
+	int i, j, k;
+	const char **ids;
+	int idnum;
+	int ret = 0;
+	struct expr_parse_ctx ctx;
+	double result;
+
+	i = 0;
+	for (;;) {
+		map = &pmu_events_map[i++];
+		if (!map->table) {

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 450

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+			map = NULL;
+			break;
+		}
+		j = 0;
+		for (;;) {
+			pe = &map->table[j++];
+			if (!pe->name && !pe->metric_group && !pe->metric_name)
+				break;
+			if (!pe->metric_expr)
+				continue;

we could probably make a for_each_metric_helper(), as this code seems duplicated a few times (like metricgroup__add_netric())

Quoted 19 lines
tools/perf/tests/pmu-events.c +19 -0
+			if (expr__find_other(pe->metric_expr, NULL,
+						&ids, &idnum, 0) < 0) {
+				pr_debug("Parse other failed for map %s %s %s\n",
+					map->cpuid, map->version, map->type);
+				pr_debug("On metric %s\n", pe->metric_name);
+				pr_debug("On expression %s\n", pe->metric_expr);
+				ret++;
+				continue;
+			}
+			expr__ctx_init(&ctx);
+
+			/*
+			 * Add all ids with a made up value. The value may
+			 * trigger divide by zero when subtracted and so try to
+			 * make them unique.
+			 */
+			for (k = 0; k < idnum; k++)
+				expr__add_id(&ctx, ids[k], k + 1);
+

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 479

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+			for (k = 0; k < idnum; k++) {
+				if (check_parse_id(ids[k], map == cpus_map, pe))
+					ret++;
+			}
+
+			if (expr__parse(&result, &ctx, pe->metric_expr, 0)) {
+				pr_debug("Parse failed for map %s %s %s\n",
+					map->cpuid, map->version, map->type);
+				pr_debug("On metric %s\n", pe->metric_name);
+				pr_debug("On expression %s\n", pe->metric_expr);

this code could be factored out a lot, see about 20 lines above

Quoted 17 lines
tools/perf/tests/pmu-events.c +17 -0
+				ret++;
+			}
+			for (k = 0; k < idnum; k++)
+				zfree(&ids[k]);
+			free(ids);
+		}
+	}
+	return ret;
+}
+
+static const struct {
+	int (*func)(void);
+	const char *desc;
+} pmu_events_testcase_table[] = {
+	{
+		.func = test_pmu_event_table,
+		.desc = "PMU event table sanity",

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 506

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+	},
+	{
+		.func = test_aliases,
+		.desc = "PMU event map aliases",
+	},
+	{
+		.func = test_parsing,
+		.desc = "Parsing of PMU event table metrics",
+	},
+};

I assume "perf test 10" will run all testcases, right?

Quoted 33 lines
tools/perf/tests/pmu-events.c +19 -0
+
+const char *test__pmu_events_subtest_get_desc(int i)
+{
+	if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+		return NULL;
+	return pmu_events_testcase_table[i].desc;
+}
+
+int test__pmu_events_subtest_get_nr(void)
+{
+	return (int)ARRAY_SIZE(pmu_events_testcase_table);
+}
+
+int test__pmu_events(struct test *test __maybe_unused, int i)
+{
+	if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+		return TEST_FAIL;
+	return pmu_events_testcase_table[i].func();
+}
tools/perf/tests/tests.h +2 -0
index d6d4ac34eeb7..8e316c30ed3c 100644
@@ -50,6 +50,8 @@ int test__perf_evsel__tp_sched_test(struct test *test, int subtest);
  int test__syscall_openat_tp_fields(struct test *test, int subtest);
  int test__pmu(struct test *test, int subtest);
  int test__pmu_events(struct test *test, int subtest);
+const char *test__pmu_events_subtest_get_desc(int subtest);
+int test__pmu_events_subtest_get_nr(void);
  int test__attr(struct test *test, int subtest);
  int test__dso_data(struct test *test, int subtest);
  int test__dso_data_cache(struct test *test, int subtest);
author reviewer
#4 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 16 lines · John Garry
Quoted 4 lines · Ian Rogers

Break pmu-events test into 2 and add a test to verify that all pmu metric expressions simply parse. Try to parse all metric ids/events, failing if metrics for the current architecture fail to parse.

Quoted 9 lines

Tested on power9, skylakex, haswell, broadwell, westmere, sandybridge and ivybridge with the patch set in place.
May fail on other architectures if metrics are invalid. In particular s390 is untested, but its expressions are trivial. The event encodings could be wrong. The other untested architectures with expressions are power8, cascadelakex, tremontx, skylake, jaketown, ivytown and variants of haswell and broadwell. For these the expression encoding is valid but the event encodings may not be.

Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid. I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

Quoted 66 lines
Quoted 66 lines
---
  tools/perf/tests/builtin-test.c |   5 +
  tools/perf/tests/pmu-events.c   | 158 ++++++++++++++++++++++++++++++--
  tools/perf/tests/tests.h        |   2 +
  3 files changed, 159 insertions(+), 6 deletions(-)
tools/perf/tests/builtin-test.c +5 -0
index 3471ec52ea11..8147c17c71ab 100644
@@ -75,6 +75,11 @@ static struct test generic_tests[] = {
      {
              .desc = "PMU events",
              .func = test__pmu_events,
+             .subtest = {
+                     .get_nr         = test__pmu_events_subtest_get_nr,
+                     .get_desc       = test__pmu_events_subtest_get_desc,
+             },
+
      },
      {
              .desc = "DSO data read",
tools/perf/tests/pmu-events.c +7 -6
index d64261da8bf7..c18b9ce8cace 100644
@@ -8,6 +8,10 @@
  #include <linux/zalloc.h>
  #include "debug.h"
  #include "../pmu-events/pmu-events.h"
+#include "util/evlist.h"
+#include "util/expr.h"
+#include "util/parse-events.h"
+#include <ctype.h>
 
  struct perf_pmu_test_event {
      struct pmu_event event;
@@ -144,7 +148,7 @@ static struct pmu_events_map *__test_pmu_get_events_map(void)
  }
 
  /* Verify generated events from pmu-events.c is as expected */
-static int __test_pmu_event_table(void)
+static int test_pmu_event_table(void)
  {
      struct pmu_events_map *map = __test_pmu_get_events_map();
      struct pmu_event *table;
@@ -347,14 +351,11 @@ static int __test__pmu_event_aliases(char *pmu_name, int *count)
      return res;
  }
 
-int test__pmu_events(struct test *test __maybe_unused,
-                  int subtest __maybe_unused)
+
+static int test_aliases(void)
  {
      struct perf_pmu *pmu = NULL;
 
-     if (__test_pmu_event_table())
-             return -1;
-
      while ((pmu = perf_pmu__scan(pmu)) != NULL) {
              int count = 0;
 
@@ -377,3 +378,148 @@ int test__pmu_events(struct test *test __maybe_unused,
 
Quoted 16 lines

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 391

Quoted 13 lines
tools/perf/tests/pmu-events.c +11 -0
      return 0;
  }
+
+static bool is_number(const char *str)
+{
+     size_t i;
+
+     for (i = 0; i < strlen(str); i++) {
+             if (!isdigit(str[i]) && str[i] != '.')
+                     return false;
+     }
+     return true;
+}

if there is no helper already for this, is this really the best place to put it? Maybe others will need it in future.

Agreed, I don't mind suggestions to put it somewhere else but making it non-hidden here wouldn't make sense. I did look for similar functionality that already existed but didn't find it.

Quoted 14 lines
Quoted 14 lines
tools/perf/tests/pmu-events.c +14 -0
+
+static int check_parse_id(const char *id, bool same_cpu, struct pmu_event *pe)
+{
+     struct parse_events_error error;
+     struct evlist *evlist;
+     int ret;
+
+     /* Numbers are always valid. */
+     if (is_number(id))
+             return 0;
+
+     evlist = evlist__new();
+     memset(&error, 0, sizeof(error));
+     ret = parse_events(evlist, id, &error);
Quoted 8 lines

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 411

Quoted 6 lines
tools/perf/tests/pmu-events.c +6 -0
+     if (ret && same_cpu) {
+             fprintf(stderr,
+                     "\nWARNING: Parse event failed metric '%s' id '%s' expr '%s'\n",
+                     pe->metric_name, id, pe->metric_expr);
+             fprintf(stderr, "Error string '%s' help '%s'\n",
+                     error.str, error.help);

do you really need to spill lines like this?

Yeah, I've installed the "kernel" c-style into my emacs and I get this. I do manually fix indentation often, but I suspect I was fight 80 columns and checkpatch.pl here (outside of "kernel" development clang-format rocks my world as it puts an end to any bike shedding - if you don't like the layout fix the format file).

; Kernel formatting, see:
; https://www.kernel.org/doc/html/v4.10/process/coding-style.html
(defun c-lineup-arglist-tabs-only (ignored)
"Line up argument lists by tabs, not spaces"
(let* ((anchor (c-langelem-pos c-syntactic-element))

         (column (c-langelem-2nd-pos c-syntactic-element))
         (offset (- (1+ column) anchor))
         (steps (floor offset c-basic-offset)))
    (* (max steps 1)
       c-basic-offset)))

(add-hook 'c-mode-common-hook

          (lambda ()
            ;; Add kernel style
            (c-add-style
             "linux-tabs-only"
             '("linux" (c-offsets-alist
                        (arglist-cont-nonempty
                         c-lineup-gcc-asm-reg
                         c-lineup-arglist-tabs-only))))))

(add-hook 'c-mode-hook
(lambda ()
(let ((filename (buffer-file-name)))
;; Enable kernel mode for the appropriate files (when (and filename
(string-match (expand-file-name "~/kernel-trees")
filename))
(setq indent-tabs-mode t)
(setq show-trailing-whitespace t)
(c-set-style "linux-tabs-only")))))

Quoted 34 lines
Quoted 34 lines
tools/perf/tests/pmu-events.c +34 -0
+     } else if (ret) {
+             pr_debug3("Parse event failed, but for an event that may not be supported by this CPU.\nid '%s' metric '%s' expr '%s'\n",
+                     id, pe->metric_name, pe->metric_expr);
+     }
+     evlist__delete(evlist);
+     free(error.str);
+     free(error.help);
+     free(error.first_str);
+     free(error.first_help);
+     /* TODO: too many metrics are broken to fail on this test currently. */
+     return 0;
+}
+
+static int test_parsing(void)
+{
+     struct pmu_events_map *cpus_map = perf_pmu__find_map(NULL);
+     struct pmu_events_map *map;
+     struct pmu_event *pe;
+     int i, j, k;
+     const char **ids;
+     int idnum;
+     int ret = 0;
+     struct expr_parse_ctx ctx;
+     double result;
+
+     i = 0;
+     for (;;) {
+             map = &pmu_events_map[i++];
+             if (!map->table) {
+                     map = NULL;
+                     break;
+             }
+             j = 0;
+             for (;;) {
Quoted 8 lines

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 450

Quoted 5 lines
tools/perf/tests/pmu-events.c +5 -0
+                     pe = &map->table[j++];
+                     if (!pe->name && !pe->metric_group && !pe->metric_name)
+                             break;
+                     if (!pe->metric_expr)
+                             continue;

we could probably make a for_each_metric_helper(), as this code seems duplicated a few times (like metricgroup__add_netric())

Sounds like useful follow-up clean up.

Quoted 23 lines
Quoted 23 lines
tools/perf/tests/pmu-events.c +23 -0
+                     if (expr__find_other(pe->metric_expr, NULL,
+                                             &ids, &idnum, 0) < 0) {
+                             pr_debug("Parse other failed for map %s %s %s\n",
+                                     map->cpuid, map->version, map->type);
+                             pr_debug("On metric %s\n", pe->metric_name);
+                             pr_debug("On expression %s\n", pe->metric_expr);
+                             ret++;
+                             continue;
+                     }
+                     expr__ctx_init(&ctx);
+
+                     /*
+                      * Add all ids with a made up value. The value may
+                      * trigger divide by zero when subtracted and so try to
+                      * make them unique.
+                      */
+                     for (k = 0; k < idnum; k++)
+                             expr__add_id(&ctx, ids[k], k + 1);
+
+                     for (k = 0; k < idnum; k++) {
+                             if (check_parse_id(ids[k], map == cpus_map, pe))
+                                     ret++;
+                     }
Quoted 8 lines

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 479

Quoted 6 lines
tools/perf/tests/pmu-events.c +6 -0
+
+                     if (expr__parse(&result, &ctx, pe->metric_expr, 0)) {
+                             pr_debug("Parse failed for map %s %s %s\n",
+                                     map->cpuid, map->version, map->type);
+                             pr_debug("On metric %s\n", pe->metric_name);
+                             pr_debug("On expression %s\n", pe->metric_expr);

this code could be factored out a lot, see about 20 lines above

Ack.

Quoted 21 lines
Quoted 21 lines
tools/perf/tests/pmu-events.c +21 -0
+                             ret++;
+                     }
+                     for (k = 0; k < idnum; k++)
+                             zfree(&ids[k]);
+                     free(ids);
+             }
+     }
+     return ret;
+}
+
+static const struct {
+     int (*func)(void);
+     const char *desc;
+} pmu_events_testcase_table[] = {
+     {
+             .func = test_pmu_event_table,
+             .desc = "PMU event table sanity",
+     },
+     {
+             .func = test_aliases,
+             .desc = "PMU event map aliases",
Quoted 8 lines

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 506

Quoted 6 lines
tools/perf/tests/pmu-events.c +6 -0
+     },
+     {
+             .func = test_parsing,
+             .desc = "Parsing of PMU event table metrics",
+     },
+};

I assume "perf test 10" will run all testcases, right?

Yes. Doing it this way breaks out which test fails.

Quoted 34 lines
Quoted 33 lines
tools/perf/tests/pmu-events.c +19 -0
+
+const char *test__pmu_events_subtest_get_desc(int i)
+{
+     if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+             return NULL;
+     return pmu_events_testcase_table[i].desc;
+}
+
+int test__pmu_events_subtest_get_nr(void)
+{
+     return (int)ARRAY_SIZE(pmu_events_testcase_table);
+}
+
+int test__pmu_events(struct test *test __maybe_unused, int i)
+{
+     if (i < 0 || i >= (int)ARRAY_SIZE(pmu_events_testcase_table))
+             return TEST_FAIL;
+     return pmu_events_testcase_table[i].func();
+}
tools/perf/tests/tests.h +2 -0
index d6d4ac34eeb7..8e316c30ed3c 100644
@@ -50,6 +50,8 @@ int test__perf_evsel__tp_sched_test(struct test *test, int subtest);
  int test__syscall_openat_tp_fields(struct test *test, int subtest);
  int test__pmu(struct test *test, int subtest);
  int test__pmu_events(struct test *test, int subtest);
+const char *test__pmu_events_subtest_get_desc(int subtest);
+int test__pmu_events_subtest_get_nr(void);
  int test__attr(struct test *test, int subtest);
  int test__dso_data(struct test *test, int subtest);
  int test__dso_data_cache(struct test *test, int subtest);
#5 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 5 lines · Ian Rogers
Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid.

So I think that has some value. I mean, just to detect syntax errors, like those remedied in "perf metrics: fix parse errors in power8 metrics".

Quoted 3 lines

I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

But we could at least check that there are event aliases for that CPU, right? (by examining the JSONs for that cpu). If the event alias does not actually match on the target CPU, then that can't be helped.

Cheers,
John

author reviewer
#6 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 11 lines · John Garry
Quoted 5 lines · Ian Rogers
Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid.

So I think that has some value. I mean, just to detect syntax errors, like those remedied in "perf metrics: fix parse errors in power8 metrics".

Quoted 7 lines
Quoted 3 lines

I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

But we could at least check that there are event aliases for that CPU, right? (by examining the JSONs for that cpu). If the event alias does not actually match on the target CPU, then that can't be helped.

Agreed, I think there will be some cases where something more can be done. Jiri has proposed fake pmus as well:
https://www.spinics.net/lists/linux-perf-users/msg11760.html
I don't know how much sense it makes trying to get this in jevents, as long as 'perf test' is run.

Thanks,
Ian

Quoted 3 lines

Cheers,
John

#7 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 20 lines · Ian Rogers
Quoted 11 lines · John Garry
Quoted 5 lines · Ian Rogers
Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid.

So I think that has some value. I mean, just to detect syntax errors, like those remedied in "perf metrics: fix parse errors in power8 metrics".

Quoted 7 lines
Quoted 3 lines

I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

But we could at least check that there are event aliases for that CPU, right? (by examining the JSONs for that cpu). If the event alias does not actually match on the target CPU, then that can't be helped.

Quoted 5 lines

Agreed, I think there will be some cases where something more can be done. Jiri has proposed fake pmus as well:
https://www.spinics.net/lists/linux-perf-users/msg11760.html
I don't know how much sense it makes trying to get this in jevents, as long as 'perf test' is run.

At a glance, that does not look like something we would want in jevents. But rather the metric expr parsing error detection and alias checking.

About jirka's patch:

tools/perf/tests/pmu-events.c +7 -0
@@ -485,6 +485,102 @@ static int test_parsing(void)
 	return ret == 0 ? TEST_OK : TEST_SKIP;
 }
 
+
+static struct test_metric metrics[] = {
+	{ .metric = "imx8_ddr0@read\\-cycles@ * 4 * 4", },
+	{ .metric = "imx8_ddr0@axid\\-read\\,axi_mask\\=0xffff\\,axi_id\\=0x0000@ * 4", },
+	{ .metric = "(cstate_pkg@c2\\-residency@ / msr@tsc@) * 100", },
+	{ .metric = "(imx8_ddr0@read\\-cycles@ + imx8_ddr0@write\\-cycles@)", },
+};
 

Maybe we could add these to pmu-events/arch/test/test_cpu/metric.json, and get at them that way.

Thanks,
John

#8 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 47 lines · John Garry
Quoted 20 lines · Ian Rogers
Quoted 11 lines · John Garry
Quoted 5 lines · Ian Rogers
Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid.

So I think that has some value. I mean, just to detect syntax errors, like those remedied in "perf metrics: fix parse errors in power8 metrics".

Quoted 7 lines
Quoted 3 lines

I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

But we could at least check that there are event aliases for that CPU, right? (by examining the JSONs for that cpu). If the event alias does not actually match on the target CPU, then that can't be helped.

Quoted 5 lines

Agreed, I think there will be some cases where something more can be done. Jiri has proposed fake pmus as well:
https://www.spinics.net/lists/linux-perf-users/msg11760.html
I don't know how much sense it makes trying to get this in jevents, as long as 'perf test' is run.

At a glance, that does not look like something we would want in jevents. But rather the metric expr parsing error detection and alias checking.

About jirka's patch:

tools/perf/tests/pmu-events.c +7 -0
@@ -485,6 +485,102 @@ static int test_parsing(void)
 	return ret == 0 ? TEST_OK : TEST_SKIP;
 }
 
+
+static struct test_metric metrics[] = {
+	{ .metric = "imx8_ddr0@read\\-cycles@ * 4 * 4", },
+	{ .metric = "imx8_ddr0@axid\\-read\\,axi_mask\\=0xffff\\,axi_id\\=0x0000@
 * 4", },
+	{ .metric = "(cstate_pkg@c2\\-residency@ / msr@tsc@) * 100", },
+	{ .metric = "(imx8_ddr0@read\\-cycles@ + imx8_ddr0@write\\-cycles@)", },
+};
 
Quoted 2 lines

Maybe we could add these to pmu-events/arch/test/test_cpu/metric.json, and get at them that way.

that test sets the 'fake pmu' stuff.. could we do that in your test?

jirka

#9 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

Quoted 48 lines · Jiri Olsa
Quoted 47 lines · John Garry
Quoted 20 lines · Ian Rogers
Quoted 11 lines · John Garry
Quoted 5 lines · Ian Rogers
Quoted 2 lines

Out of interest, if we could move the validation of metrics to jevents, how much functionality would we still have here?

If we add checking to jevents then the MetricExpr would be known to be valid, however, the events (aka ids) within the expression could be invalid.

So I think that has some value. I mean, just to detect syntax errors, like those remedied in "perf metrics: fix parse errors in power8 metrics".

Quoted 7 lines
Quoted 3 lines

I'm not sure we could realistically check the events at jevents (build) time as there is no guarantee that the machine we run on is the same as the one we compile on.

But we could at least check that there are event aliases for that CPU, right? (by examining the JSONs for that cpu). If the event alias does not actually match on the target CPU, then that can't be helped.

Quoted 5 lines

Agreed, I think there will be some cases where something more can be done. Jiri has proposed fake pmus as well:
https://www.spinics.net/lists/linux-perf-users/msg11760.html
I don't know how much sense it makes trying to get this in jevents, as long as 'perf test' is run.

At a glance, that does not look like something we would want in jevents. But rather the metric expr parsing error detection and alias checking.

About jirka's patch:

tools/perf/tests/pmu-events.c +7 -0
@@ -485,6 +485,102 @@ static int test_parsing(void)
  	return ret == 0 ? TEST_OK : TEST_SKIP;
  }
 
+
+static struct test_metric metrics[] = {
+	{ .metric = "imx8_ddr0@read\\-cycles@ * 4 * 4", },
+	{ .metric = "imx8_ddr0@axid\\-read\\,axi_mask\\=0xffff\\,axi_id\\=0x0000@
 * 4", },
+	{ .metric = "(cstate_pkg@c2\\-residency@ / msr@tsc@) * 100", },
+	{ .metric = "(imx8_ddr0@read\\-cycles@ + imx8_ddr0@write\\-cycles@)", },
+};
 
Quoted 4 lines
Quoted 2 lines

Maybe we could add these to pmu-events/arch/test/test_cpu/metric.json, and get at them that way.

that test sets the 'fake pmu' stuff.. could we do that in your test?

I'm not sure about a test comparable to the alias match testing, but at least it should be possible to verify that jevents generates as-expected metric events (as done in __test_pmu_event_table() for regular events).

Cheers,
John

#10 lore raw

Re: [PATCH 2/2] perf test: Improve pmu event metric testing

In reply to Ian Rogers

On Tue, May 12, 2020 at 11:22:36PM -0700, Ian Rogers wrote:

SNIP

Quoted 14 lines
tools/perf/tests/pmu-events.c +14 -0
+
+static int check_parse_id(const char *id, bool same_cpu, struct pmu_event *pe)
+{
+	struct parse_events_error error;
+	struct evlist *evlist;
+	int ret;
+
+	/* Numbers are always valid. */
+	if (is_number(id))
+		return 0;
+
+	evlist = evlist__new();
+	memset(&error, 0, sizeof(error));
+	ret = parse_events(evlist, id, &error);

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 415

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+	if (ret && same_cpu) {
+		fprintf(stderr,
+			"\nWARNING: Parse event failed metric '%s' id '%s' expr '%s'\n",
+			pe->metric_name, id, pe->metric_expr);
+		fprintf(stderr, "Error string '%s' help '%s'\n",
+			error.str, error.help);
+	} else if (ret) {
+		pr_debug3("Parse event failed, but for an event that may not be supported by this CPU.\nid '%s' metric '%s' expr '%s'\n",
+			id, pe->metric_name, pe->metric_expr);
+	}

I wonder if we could add 'fake pmu' that would be used for tests and use it parse_events_add_pmu to add 'fake' evsel but the name would be correct.. we could add parse_events_state::fake_pmu bool for that

rest of the event types (other than pmu syntax) might be ok

Quoted 16 lines
tools/perf/tests/pmu-events.c +16 -0
+	evlist__delete(evlist);
+	free(error.str);
+	free(error.help);
+	free(error.first_str);
+	free(error.first_help);
+	/* TODO: too many metrics are broken to fail on this test currently. */
+	return 0;
+}
+
+static int test_parsing(void)
+{
+	struct pmu_events_map *cpus_map = perf_pmu__find_map(NULL);
+	struct pmu_events_map *map;
+	struct pmu_event *pe;
+	int i, j, k;
+	const char **ids;

tools/perf/tests/pmu-events.c int test__pmu_events(struct test *test __maybe_unused, · line 441

Quoted 10 lines
tools/perf/tests/pmu-events.c +10 -0
+	int idnum;
+	int ret = 0;
+	struct expr_parse_ctx ctx;
+	double result;
+
+	i = 0;
+	for (;;) {
+		map = &pmu_events_map[i++];
+		if (!map->table) {
+			map = NULL;

hum, what's the map = NULL for in here?

thanks,
jirka

Quoted 46 lines
tools/perf/tests/pmu-events.c +46 -0
+			break;
+		}
+		j = 0;
+		for (;;) {
+			pe = &map->table[j++];
+			if (!pe->name && !pe->metric_group && !pe->metric_name)
+				break;
+			if (!pe->metric_expr)
+				continue;
+			if (expr__find_other(pe->metric_expr, NULL,
+						&ids, &idnum, 0) < 0) {
+				pr_debug("Parse other failed for map %s %s %s\n",
+					map->cpuid, map->version, map->type);
+				pr_debug("On metric %s\n", pe->metric_name);
+				pr_debug("On expression %s\n", pe->metric_expr);
+				ret++;
+				continue;
+			}
+			expr__ctx_init(&ctx);
+
+			/*
+			 * Add all ids with a made up value. The value may
+			 * trigger divide by zero when subtracted and so try to
+			 * make them unique.
+			 */
+			for (k = 0; k < idnum; k++)
+				expr__add_id(&ctx, ids[k], k + 1);
+
+			for (k = 0; k < idnum; k++) {
+				if (check_parse_id(ids[k], map == cpus_map, pe))
+					ret++;
+			}
+
+			if (expr__parse(&result, &ctx, pe->metric_expr, 0)) {
+				pr_debug("Parse failed for map %s %s %s\n",
+					map->cpuid, map->version, map->type);
+				pr_debug("On metric %s\n", pe->metric_name);
+				pr_debug("On expression %s\n", pe->metric_expr);
+				ret++;
+			}
+			for (k = 0; k < idnum; k++)
+				zfree(&ids[k]);
+			free(ids);
+		}
+	}
+	return ret;

SNIP