perf expr: Add flex scanner
Jiri Olsa
reviewercover perf expr: Add flex scanner
hi,
while preparing changes for user defined metric expressions I also moved the expression manual parser to flex.
The reason is to have an easy and reasonable way to support and parse multiple user-defined metric expressions from command line or file.
I was posponing the change, but I just saw another update to the expr manual scanner (from Kajol Jain), so cherry picked just the expr flex code changes to get it out.
Kajol Jain,
I think it should ease up your change for unknown values marked by '?'. Would you consider rebasing your changes on top of this?
Available also in:
git://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git
perf/metric_flex
thanks,
jirka
---
Jiri Olsa (4):
perf expr: Add expr.c object
perf expr: Move expr lexer to flex
perf expr: Increase EXPR_MAX_OTHER
perf expr: Straighten expr__parse/expr__find_other interface
tools/perf/tests/expr.c | 6 ++-- tools/perf/util/Build | 11 +++++++- tools/perf/util/expr.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/perf/util/expr.h | 8 ++---- tools/perf/util/expr.l | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ tools/perf/util/expr.y | 185 +++++++++++++++++++++---------------------------------------------------------------------------------------------------- tools/perf/util/stat-shadow.c | 4 +-- 7 files changed, 242 insertions(+), 166 deletions(-) create mode 100644 tools/perf/util/expr.c create mode 100644 tools/perf/util/expr.l
Jiri Olsa
reviewerPATCH 1/4 perf expr: Add expr.c object
Add generic expr code into new expr.c object.
The expr.c object will be mainly used in following change that will get rid of the manual flex code,
--- tools/perf/util/Build | 1 + tools/perf/util/expr.c | 20 ++++++++++++++++++++ tools/perf/util/expr.y | 16 ---------------- 3 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 tools/perf/util/expr.c
tools/perf/util/Build +1 -0
@@ -122,6 +122,7 @@ perf-y += vsprintf.o perf-y += units.o perf-y += time-utils.o perf-y += expr-bison.o +perf-y += expr.o perf-y += branch.o perf-y += mem2node.o
tools/perf/util/expr.c new file +20 -0
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <assert.h> +#include "expr.h" + +/* Caller must make sure id is allocated */ +void expr__add_id (struct parse_ctx *ctx,const char *name,double val) +{ +int idx; + +assert (ctx->num_ids <MAX_PARSE_ID ); + idx = ctx->num_ids++; + ctx->ids[idx].name = name; + ctx->ids[idx].val = val; +} + +void expr__ctx_init (struct parse_ctx *ctx) +{ + ctx->num_ids =0 ; +} +
tools/perf/util/expr.y +0 -16
@@ -6,7 +6,6 @@ #define IN_EXPR_Y 1 #include "expr.h" #include "smt.h" -#include <assert.h> #include <string.h> #define MAXIDLEN 256
@@ -169,21 +168,6 @@ static int expr__lex(YYSTYPE *res, const char **pp) return tok; } -/* Caller must make sure id is allocated */ -void expr__add_id(struct parse_ctx *ctx, const char *name, double val) -{ - int idx; - assert(ctx->num_ids < MAX_PARSE_ID); - idx = ctx->num_ids++; - ctx->ids[idx].name = name; - ctx->ids[idx].val = val; -} - -void expr__ctx_init(struct parse_ctx *ctx) -{ - ctx->num_ids = 0; -} - static bool already_seen(const char *val, const char *one, const char **other, int num_other) {
Signature · 1 line
2.24.1
Jiri Olsa
reviewerPATCH 2/4 perf expr: Move expr lexer to flex
In reply to Jiri OlsaAdding expr flex code instead of the manual parser code. So it's easily extensible in upcoming changes.
The new flex code is in flex.l object and gets compiled like all the other flexers we use. It's defined as flex reentrant parser.
It's used by both expr__parse and expr__find_other interfaces by separating the starting point.
There's no intended change of functionality ;-) the test expr is passing.
--- tools/perf/util/Build | 10 ++- tools/perf/util/expr.c | 91 ++++++++++++++++++++++ tools/perf/util/expr.h | 2 - tools/perf/util/expr.l | 83 ++++++++++++++++++++ tools/perf/util/expr.y | 169 ++++++++--------------------------------- 5 files changed, 214 insertions(+), 141 deletions(-) create mode 100644 tools/perf/util/expr.l
tools/perf/util/Build +9 -1
@@ -121,6 +121,7 @@ perf-y += mem-events.o perf-y += vsprintf.o perf-y += units.o perf-y += time-utils.o +perf-y += expr-flex.o perf-y += expr-bison.o perf-y += expr.o perf-y += branch.o
@@ -190,9 +191,13 @@ $(OUTPUT)util/parse-events-bison.c: util/parse-events.y $(call rule_mkdir) $(Q)$(call echo-cmd,bison)$(BISON) -v util/parse-events.y -d $(PARSER_DEBUG_BISON) -o $@ -p parse_events_ +$(OUTPUT)util/expr-flex.c: util/expr.l $(OUTPUT)util/expr-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $@ --header-file=$(OUTPUT)util/expr-flex.h $(PARSER_DEBUG_FLEX) util/expr.l + $(OUTPUT)util/expr-bison.c: util/expr.y $(call rule_mkdir) - $(Q)$(call echo-cmd,bison)$(BISON) -v util/expr.y -d $(PARSER_DEBUG_BISON) -o $@ -p expr__ + $(Q)$(call echo-cmd,bison)$(BISON) -v util/expr.y -d $(PARSER_DEBUG_BISON) -o $@ -p expr_ $(OUTPUT)util/pmu-flex.c: util/pmu.l $(OUTPUT)util/pmu-bison.c $(call rule_mkdir)
@@ -204,12 +209,14 @@ $(OUTPUT)util/pmu-bison.c: util/pmu.y CFLAGS_parse-events-flex.o += -w CFLAGS_pmu-flex.o += -w +CFLAGS_expr-flex.o += -w CFLAGS_parse-events-bison.o += -DYYENABLE_NLS=0 -w CFLAGS_pmu-bison.o += -DYYENABLE_NLS=0 -DYYLTYPE_IS_TRIVIAL=0 -w CFLAGS_expr-bison.o += -DYYENABLE_NLS=0 -DYYLTYPE_IS_TRIVIAL=0 -w $(OUTPUT)util/parse-events.o: $(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-bison.c $(OUTPUT)util/pmu.o: $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-bison.c +$(OUTPUT)util/expr.o: $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-bison.c CFLAGS_bitmap.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" CFLAGS_find_bit.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))"
@@ -217,6 +224,7 @@ CFLAGS_rbtree.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ET CFLAGS_libstring.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" CFLAGS_hweight.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" CFLAGS_parse-events.o += -Wno-redundant-decls +CFLAGS_expr.o += -Wno-redundant-decls CFLAGS_header.o += -include $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)util/kallsyms.o: ../lib/symbol/kallsyms.c FORCE
tools/perf/util/expr.c +91 -0
100 lines not on this page · raw message
tools/perf/util/expr.h +0 -2
@@ -17,9 +17,7 @@ struct parse_ctx {void expr__ctx_init (struct parse_ctx *ctx);void expr__add_id (struct parse_ctx *ctx,const char *id,double val); -#ifndef IN_EXPR_Y int expr__parse (double *final_val,struct parse_ctx *ctx,const char **pp); -#endif int expr__find_other (const char *p,const char *one,const char ***other,int *num_other);
tools/perf/util/expr.l new file +83 -0
83 lines not on this page · raw message
tools/perf/util/expr.y +31 -138
207 lines not on this page · raw message
Signature · 1 line
2.24.1
Re: [PATCH 2/4] perf expr: Move expr lexer to flex
Quoted 8 lines
tools/perf/util/expr.l new file +8
-0+max { return MAX; } +min { return MIN; } +if { return IF; } +else { return ELSE; } +#smt_on { return SMT_ON; } +{number} { return value(yyscanner, 10); } +{symbol} { return str(yyscanner, ID); } +"|" { return '|'; }
tools/perf/util/expr.l line 76
Quoted 10 lines
tools/perf/util/expr.l new file +10
-0+"^" { return '^'; } +"&" { return '&'; } +"-" { return '-'; } +"+" { return '+'; } +"*" { return '*'; } +"/" { return '/'; } +"%" { return '%'; } +"(" { return '('; } +")" { return ')'; } +"," { return ','; }
Didn't think there was a comma, but ok.
Looks reasonable to me.
-Andi
Re: [PATCH 2/4] perf expr: Move expr lexer to flex
Quoted 21 lines · Andi Kleen
Quoted 8 lines
tools/perf/util/expr.l new file +8
-0+max { return MAX; } +min { return MIN; } +if { return IF; } +else { return ELSE; } +#smt_on { return SMT_ON; } +{number} { return value(yyscanner, 10); } +{symbol} { return str(yyscanner, ID); } +"|" { return '|'; }tools/perf/util/expr.l line 76
Quoted 10 lines
tools/perf/util/expr.l new file +10
-0+"^" { return '^'; } +"&" { return '&'; } +"-" { return '-'; } +"+" { return '+'; } +"*" { return '*'; } +"/" { return '/'; } +"%" { return '%'; } +"(" { return '('; } +")" { return ')'; } +"," { return ','; }Didn't think there was a comma, but ok.
Quoted 1 line
Looks reasonable to me.
it still needs to replace/skip special chars,
I'll post fixed version next week
thanks,
jirka
Re: [PATCH 2/4] perf expr: Move expr lexer to flex
In reply to Andi KleenQuoted 12 lines · Andi Kleen
Quoted 12 lines
tools/perf/util/expr.l new file +12
-0+max { return MAX; } +min { return MIN; } +if { return IF; } +else { return ELSE; } +#smt_on { return SMT_ON; } +{number} { return value(yyscanner, 10); } +{symbol} { return str(yyscanner, ID); } +"|" { return '|'; } +"^" { return '^'; } +"&" { return '&'; } +"-" { return '-'; } +"+" { return '+'; }
Quoted 8 lines
Quoted 6 lines
+"*" { return '*'; }
+"/" { return '/'; }
+"%" { return '%'; }
+"(" { return '('; }
+")" { return ')'; }
+"," { return ','; }Didn't think there was a comma, but ok.
actually it's for the min/max functions arguments separation I posted v2
jirka
Quoted 5 lines
Looks reasonable to me.
-Andi
Jiri Olsa
reviewerPATCH 3/4 perf expr: Increase EXPR_MAX_OTHER
In reply to Jiri OlsaThere's no need to be greedy on allowed variables, also when some of the metrics define more than 15 variables, like Branch_Misprediction_Cost.
Increasing the maximum to 100.
--- tools/perf/util/expr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/perf/util/expr.h +1 -1
@@ -2,7 +2,7 @@#ifndef PARSE_CTX_H #define PARSE_CTX_H 1 -#define EXPR_MAX_OTHER 15 +#define EXPR_MAX_OTHER 100#define MAX_PARSE_ID EXPR_MAX_OTHER struct parse_id {
Signature · 1 line
2.24.1
Jiri Olsa
reviewerPATCH 4/4 perf expr: Straighten expr__parse/expr__find_other interface
In reply to Jiri OlsaNow with flex parser we don't need to update parsed string pointer, so the interface can just passed the pointer to the expression instead of pointer to pointer.
--- tools/perf/tests/expr.c | 6 +++--- tools/perf/util/expr.c | 8 ++++---- tools/perf/util/expr.h | 4 ++-- tools/perf/util/stat-shadow.c | 4 +--- 4 files changed, 10 insertions(+), 12 deletions(-)
tools/perf/tests/expr.c +3 -3
@@ -10,7 +10,7 @@ static int test(struct parse_ctx *ctx, const char *e, double val2) {double val; -if (expr__parse (&val, ctx, &e)) +if (expr__parse (&val, ctx, e))TEST_ASSERT_VAL ("parse test failed" ,0 );TEST_ASSERT_VAL ("unexpected value" , val == val2);return 0 ;
@@ -44,11 +44,11 @@ int test__expr(struct test *t __maybe_unused, int subtest __maybe_unused)return ret; p ="FOO/0" ; - ret =expr__parse (&val, &ctx, &p); + ret =expr__parse (&val, &ctx, p);TEST_ASSERT_VAL ("division by zero" , ret ==1 ); p ="BAR/" ; - ret =expr__parse (&val, &ctx, &p); + ret =expr__parse (&val, &ctx, p);TEST_ASSERT_VAL ("missing operand" , ret ==1 ); TEST_ASSERT_VAL("find other",
tools/perf/util/expr.c +4 -4
@@ -52,9 +52,9 @@ __expr__parse(double *val, struct parse_ctx *ctx, const char *expr,return ret; } -int expr__parse (double *final_val,struct parse_ctx *ctx,const char **pp) +int expr__parse (double *final_val,struct parse_ctx *ctx,const char *expr) { -return __expr__parse (final_val, ctx, *pp,EXPR_PARSE ); +return __expr__parse (final_val, ctx, expr,EXPR_PARSE ); } static bool
@@ -71,14 +71,14 @@ already_seen(const char *val, const char *one, const char **other,return false; } -int expr__find_other (const char *p,const char *one,const char ***other, +int expr__find_other (const char *expr,const char *one,const char ***other,int *num_other) {int err, i =0 , j =0 ;struct parse_ctx ctx;expr__ctx_init (&ctx); - err =__expr__parse (NULL , &ctx, p,EXPR_OTHER ); + err =__expr__parse (NULL , &ctx, expr,EXPR_OTHER );if (err)return err;
tools/perf/util/expr.h +2 -2
@@ -17,8 +17,8 @@ struct parse_ctx {void expr__ctx_init (struct parse_ctx *ctx);void expr__add_id (struct parse_ctx *ctx,const char *id,double val); -int expr__parse (double *final_val,struct parse_ctx *ctx,const char **pp); -int expr__find_other (const char *p,const char *one,const char ***other, +int expr__parse (double *final_val,struct parse_ctx *ctx,const char *expr); +int expr__find_other (const char *expr,const char *one,const char ***other,int *num_other);#endif
tools/perf/util/stat-shadow.c +1 -3
@@ -779,9 +779,7 @@ static void generic_metric(struct perf_stat_config *config, }if (!metric_events[i]) { -const char *p = metric_expr; - -if (expr__parse (&ratio, &pctx, &p) ==0 ) { +if (expr__parse (&ratio, &pctx, metric_expr) ==0 ) {char *unit;char metric_bf[64 ];
Signature · 1 line
2.24.1