Skip to contentSkip to conversation

perf expr: Add flex scanner

RFC Superseded Jiri Olsa 3 participants 8 messages Feb 21, 2020 – Feb 24, 2020 perf events

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Jiri Olsa

author reviewer
#0 lore raw

cover perf expr: Add flex scanner

hi,
while preparing changes for user defined metric expressions I also moved the expression manual parser to flex.

The reason is to have an easy and reasonable way to support and parse multiple user-defined metric expressions from command line or file.

I was posponing the change, but I just saw another update to the expr manual scanner (from Kajol Jain), so cherry picked just the expr flex code changes to get it out.

Kajol Jain,
I think it should ease up your change for unknown values marked by '?'. Would you consider rebasing your changes on top of this?

Available also in:
git://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git
perf/metric_flex

thanks,
jirka

---
Jiri Olsa (4):
perf expr: Add expr.c object
perf expr: Move expr lexer to flex
perf expr: Increase EXPR_MAX_OTHER
perf expr: Straighten expr__parse/expr__find_other interface

 tools/perf/tests/expr.c       |   6 ++--
 tools/perf/util/Build         |  11 +++++++-
 tools/perf/util/expr.c        | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/expr.h        |   8 ++----
 tools/perf/util/expr.l        |  83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/expr.y        | 185 +++++++++++++++++++++----------------------------------------------------------------------------------------------------
 tools/perf/util/stat-shadow.c |   4 +--
 7 files changed, 242 insertions(+), 166 deletions(-)
 create mode 100644 tools/perf/util/expr.c
 create mode 100644 tools/perf/util/expr.l
author reviewer
#1 lore raw

PATCH 1/4 perf expr: Add expr.c object

Add generic expr code into new expr.c object.

The expr.c object will be mainly used in following change that will get rid of the manual flex code,

---
 tools/perf/util/Build  |  1 +
 tools/perf/util/expr.c | 20 ++++++++++++++++++++
 tools/perf/util/expr.y | 16 ----------------
 3 files changed, 21 insertions(+), 16 deletions(-)
 create mode 100644 tools/perf/util/expr.c
tools/perf/util/Build +1 -0
index 07da6c790b63..6fdf073093a6 100644
@@ -122,6 +122,7 @@ perf-y += vsprintf.o
 perf-y += units.o
 perf-y += time-utils.o
 perf-y += expr-bison.o
+perf-y += expr.o
 perf-y += branch.o
 perf-y += mem2node.o
 
tools/perf/util/expr.c new file +20 -0
new file mode 100644
index 000000000000..4a82ff3bdbcc
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <assert.h>
+#include "expr.h"
+
+/* Caller must make sure id is allocated */
+void expr__add_id(struct parse_ctx *ctx, const char *name, double val)
+{
+	int idx;
+
+	assert(ctx->num_ids < MAX_PARSE_ID);
+	idx = ctx->num_ids++;
+	ctx->ids[idx].name = name;
+	ctx->ids[idx].val = val;
+}
+
+void expr__ctx_init(struct parse_ctx *ctx)
+{
+	ctx->num_ids = 0;
+}
+
tools/perf/util/expr.y +0 -16
index 7d226241f1d7..7cea8b7c3a5c 100644
@@ -6,7 +6,6 @@
 #define IN_EXPR_Y 1
 #include "expr.h"
 #include "smt.h"
-#include <assert.h>
 #include <string.h>
 
 #define MAXIDLEN 256
@@ -169,21 +168,6 @@ static int expr__lex(YYSTYPE *res, const char **pp)
 	return tok;
 }
 
-/* Caller must make sure id is allocated */
-void expr__add_id(struct parse_ctx *ctx, const char *name, double val)
-{
-	int idx;
-	assert(ctx->num_ids < MAX_PARSE_ID);
-	idx = ctx->num_ids++;
-	ctx->ids[idx].name = name;
-	ctx->ids[idx].val = val;
-}
-
-void expr__ctx_init(struct parse_ctx *ctx)
-{
-	ctx->num_ids = 0;
-}
-
 static bool already_seen(const char *val, const char *one, const char **other,
 			 int num_other)
 {
Signature · 1 line
2.24.1
author reviewer
#2 lore raw

PATCH 2/4 perf expr: Move expr lexer to flex

In reply to Jiri Olsa

Adding expr flex code instead of the manual parser code. So it's easily extensible in upcoming changes.

The new flex code is in flex.l object and gets compiled like all the other flexers we use. It's defined as flex reentrant parser.

It's used by both expr__parse and expr__find_other interfaces by separating the starting point.

There's no intended change of functionality ;-) the test expr is passing.

---
 tools/perf/util/Build  |  10 ++-
 tools/perf/util/expr.c |  91 ++++++++++++++++++++++
 tools/perf/util/expr.h |   2 -
 tools/perf/util/expr.l |  83 ++++++++++++++++++++
 tools/perf/util/expr.y | 169 ++++++++---------------------------------
 5 files changed, 214 insertions(+), 141 deletions(-)
 create mode 100644 tools/perf/util/expr.l
tools/perf/util/Build +9 -1
index 6fdf073093a6..c0cf8dff694e 100644
@@ -121,6 +121,7 @@ perf-y += mem-events.o
 perf-y += vsprintf.o
 perf-y += units.o
 perf-y += time-utils.o
+perf-y += expr-flex.o
 perf-y += expr-bison.o
 perf-y += expr.o
 perf-y += branch.o
@@ -190,9 +191,13 @@ $(OUTPUT)util/parse-events-bison.c: util/parse-events.y
 	$(call rule_mkdir)
 	$(Q)$(call echo-cmd,bison)$(BISON) -v util/parse-events.y -d $(PARSER_DEBUG_BISON) -o $@ -p parse_events_
 
+$(OUTPUT)util/expr-flex.c: util/expr.l $(OUTPUT)util/expr-bison.c
+	$(call rule_mkdir)
+	$(Q)$(call echo-cmd,flex)$(FLEX) -o $@ --header-file=$(OUTPUT)util/expr-flex.h $(PARSER_DEBUG_FLEX) util/expr.l
+
 $(OUTPUT)util/expr-bison.c: util/expr.y
 	$(call rule_mkdir)
-	$(Q)$(call echo-cmd,bison)$(BISON) -v util/expr.y -d $(PARSER_DEBUG_BISON) -o $@ -p expr__
+	$(Q)$(call echo-cmd,bison)$(BISON) -v util/expr.y -d $(PARSER_DEBUG_BISON) -o $@ -p expr_
 
 $(OUTPUT)util/pmu-flex.c: util/pmu.l $(OUTPUT)util/pmu-bison.c
 	$(call rule_mkdir)
@@ -204,12 +209,14 @@ $(OUTPUT)util/pmu-bison.c: util/pmu.y
 
 CFLAGS_parse-events-flex.o  += -w
 CFLAGS_pmu-flex.o           += -w
+CFLAGS_expr-flex.o          += -w
 CFLAGS_parse-events-bison.o += -DYYENABLE_NLS=0 -w
 CFLAGS_pmu-bison.o          += -DYYENABLE_NLS=0 -DYYLTYPE_IS_TRIVIAL=0 -w
 CFLAGS_expr-bison.o         += -DYYENABLE_NLS=0 -DYYLTYPE_IS_TRIVIAL=0 -w
 
 $(OUTPUT)util/parse-events.o: $(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-bison.c
 $(OUTPUT)util/pmu.o: $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-bison.c
+$(OUTPUT)util/expr.o: $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-bison.c
 
 CFLAGS_bitmap.o        += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))"
 CFLAGS_find_bit.o      += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))"
@@ -217,6 +224,7 @@ CFLAGS_rbtree.o        += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ET
 CFLAGS_libstring.o     += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))"
 CFLAGS_hweight.o       += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))"
 CFLAGS_parse-events.o  += -Wno-redundant-decls
+CFLAGS_expr.o          += -Wno-redundant-decls
 CFLAGS_header.o        += -include $(OUTPUT)PERF-VERSION-FILE
 
 $(OUTPUT)util/kallsyms.o: ../lib/symbol/kallsyms.c FORCE
tools/perf/util/expr.c +91 -0

100 lines not on this page · raw message

tools/perf/util/expr.h +0 -2
index 046160831f90..9332796e6649 100644
@@ -17,9 +17,7 @@ struct parse_ctx {
 
 void expr__ctx_init(struct parse_ctx *ctx);
 void expr__add_id(struct parse_ctx *ctx, const char *id, double val);
-#ifndef IN_EXPR_Y
 int expr__parse(double *final_val, struct parse_ctx *ctx, const char **pp);
-#endif
 int expr__find_other(const char *p, const char *one, const char ***other,
 		int *num_other);
 
tools/perf/util/expr.l new file +83 -0

83 lines not on this page · raw message

tools/perf/util/expr.y +31 -138

207 lines not on this page · raw message

Signature · 1 line
2.24.1
#3 lore raw

Re: [PATCH 2/4] perf expr: Move expr lexer to flex

Quoted 8 lines
tools/perf/util/expr.l new file +8 -0
+max		{ return MAX; }
+min		{ return MIN; }
+if		{ return IF; }
+else		{ return ELSE; }
+#smt_on		{ return SMT_ON; }
+{number}	{ return value(yyscanner, 10); }
+{symbol}	{ return str(yyscanner, ID); }
+"|"		{ return '|'; }

tools/perf/util/expr.l line 76

Quoted 10 lines
tools/perf/util/expr.l new file +10 -0
+"^"		{ return '^'; }
+"&"		{ return '&'; }
+"-"		{ return '-'; }
+"+"		{ return '+'; }
+"*"		{ return '*'; }
+"/"		{ return '/'; }
+"%"		{ return '%'; }
+"("		{ return '('; }
+")"		{ return ')'; }
+","		{ return ','; }

Didn't think there was a comma, but ok.

Looks reasonable to me.

-Andi

#4 lore raw

Re: [PATCH 2/4] perf expr: Move expr lexer to flex

Quoted 21 lines · Andi Kleen
Quoted 8 lines
tools/perf/util/expr.l new file +8 -0
+max		{ return MAX; }
+min		{ return MIN; }
+if		{ return IF; }
+else		{ return ELSE; }
+#smt_on		{ return SMT_ON; }
+{number}	{ return value(yyscanner, 10); }
+{symbol}	{ return str(yyscanner, ID); }
+"|"		{ return '|'; }

tools/perf/util/expr.l line 76

Quoted 10 lines
tools/perf/util/expr.l new file +10 -0
+"^"		{ return '^'; }
+"&"		{ return '&'; }
+"-"		{ return '-'; }
+"+"		{ return '+'; }
+"*"		{ return '*'; }
+"/"		{ return '/'; }
+"%"		{ return '%'; }
+"("		{ return '('; }
+")"		{ return ')'; }
+","		{ return ','; }

Didn't think there was a comma, but ok.

Quoted 1 line

Looks reasonable to me.

it still needs to replace/skip special chars,
I'll post fixed version next week

thanks,
jirka

#5 lore raw

Re: [PATCH 2/4] perf expr: Move expr lexer to flex

In reply to Andi Kleen
Quoted 12 lines · Andi Kleen
Quoted 12 lines
tools/perf/util/expr.l new file +12 -0
+max		{ return MAX; }
+min		{ return MIN; }
+if		{ return IF; }
+else		{ return ELSE; }
+#smt_on		{ return SMT_ON; }
+{number}	{ return value(yyscanner, 10); }
+{symbol}	{ return str(yyscanner, ID); }
+"|"		{ return '|'; }
+"^"		{ return '^'; }
+"&"		{ return '&'; }
+"-"		{ return '-'; }
+"+"		{ return '+'; }
Quoted 8 lines
Quoted 6 lines

+"*" { return '*'; }
+"/" { return '/'; }
+"%" { return '%'; }
+"(" { return '('; }
+")" { return ')'; }
+"," { return ','; }

Didn't think there was a comma, but ok.

actually it's for the min/max functions arguments separation I posted v2

jirka

Quoted 5 lines

Looks reasonable to me.

-Andi

author reviewer
#6 lore raw

PATCH 3/4 perf expr: Increase EXPR_MAX_OTHER

In reply to Jiri Olsa

There's no need to be greedy on allowed variables, also when some of the metrics define more than 15 variables, like Branch_Misprediction_Cost.

Increasing the maximum to 100.

---
 tools/perf/util/expr.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
tools/perf/util/expr.h +1 -1
index 9332796e6649..01e1529bdfd9 100644
@@ -2,7 +2,7 @@
 #ifndef PARSE_CTX_H
 #define PARSE_CTX_H 1
 
-#define EXPR_MAX_OTHER 15
+#define EXPR_MAX_OTHER 100
 #define MAX_PARSE_ID EXPR_MAX_OTHER
 
 struct parse_id {
Signature · 1 line
2.24.1
author reviewer
#7 lore raw

PATCH 4/4 perf expr: Straighten expr__parse/expr__find_other interface

In reply to Jiri Olsa

Now with flex parser we don't need to update parsed string pointer, so the interface can just passed the pointer to the expression instead of pointer to pointer.

---
 tools/perf/tests/expr.c       | 6 +++---
 tools/perf/util/expr.c        | 8 ++++----
 tools/perf/util/expr.h        | 4 ++--
 tools/perf/util/stat-shadow.c | 4 +---
 4 files changed, 10 insertions(+), 12 deletions(-)
tools/perf/tests/expr.c +3 -3
index 87843af4c118..755d73c86c68 100644
@@ -10,7 +10,7 @@ static int test(struct parse_ctx *ctx, const char *e, double val2)
 {
 	double val;
 
-	if (expr__parse(&val, ctx, &e))
+	if (expr__parse(&val, ctx, e))
 		TEST_ASSERT_VAL("parse test failed", 0);
 	TEST_ASSERT_VAL("unexpected value", val == val2);
 	return 0;
@@ -44,11 +44,11 @@ int test__expr(struct test *t __maybe_unused, int subtest __maybe_unused)
 		return ret;
 
 	p = "FOO/0";
-	ret = expr__parse(&val, &ctx, &p);
+	ret = expr__parse(&val, &ctx, p);
 	TEST_ASSERT_VAL("division by zero", ret == 1);
 
 	p = "BAR/";
-	ret = expr__parse(&val, &ctx, &p);
+	ret = expr__parse(&val, &ctx, p);
 	TEST_ASSERT_VAL("missing operand", ret == 1);
 
 	TEST_ASSERT_VAL("find other",
tools/perf/util/expr.c +4 -4
index b15cfa7ba694..6bd1e93a12b5 100644
@@ -52,9 +52,9 @@ __expr__parse(double *val, struct parse_ctx *ctx, const char *expr,
 	return ret;
 }
 
-int expr__parse(double *final_val, struct parse_ctx *ctx, const char **pp)
+int expr__parse(double *final_val, struct parse_ctx *ctx, const char *expr)
 {
-	return __expr__parse(final_val, ctx, *pp, EXPR_PARSE);
+	return __expr__parse(final_val, ctx, expr, EXPR_PARSE);
 }
 
 static bool
@@ -71,14 +71,14 @@ already_seen(const char *val, const char *one, const char **other,
 	return false;
 }
 
-int expr__find_other(const char *p, const char *one, const char ***other,
+int expr__find_other(const char *expr, const char *one, const char ***other,
 		     int *num_other)
 {
 	int err, i = 0, j = 0;
 	struct parse_ctx ctx;
 
 	expr__ctx_init(&ctx);
-	err = __expr__parse(NULL, &ctx, p, EXPR_OTHER);
+	err = __expr__parse(NULL, &ctx, expr, EXPR_OTHER);
 	if (err)
 		return err;
 
tools/perf/util/expr.h +2 -2
index 01e1529bdfd9..28a0a2dfbd8c 100644
@@ -17,8 +17,8 @@ struct parse_ctx {
 
 void expr__ctx_init(struct parse_ctx *ctx);
 void expr__add_id(struct parse_ctx *ctx, const char *id, double val);
-int expr__parse(double *final_val, struct parse_ctx *ctx, const char **pp);
-int expr__find_other(const char *p, const char *one, const char ***other,
+int expr__parse(double *final_val, struct parse_ctx *ctx, const char *expr);
+int expr__find_other(const char *expr, const char *one, const char ***other,
 		int *num_other);
 
 #endif
tools/perf/util/stat-shadow.c +1 -3
index 2c41d47f6f83..854d6abf2993 100644
@@ -779,9 +779,7 @@ static void generic_metric(struct perf_stat_config *config,
 	}
 
 	if (!metric_events[i]) {
-		const char *p = metric_expr;
-
-		if (expr__parse(&ratio, &pctx, &p) == 0) {
+		if (expr__parse(&ratio, &pctx, metric_expr) == 0) {
 			char *unit;
 			char metric_bf[64];
 
Signature · 1 line
2.24.1