A counter-proposal for the literals transition
cover A counter-proposal for the literals transition
So I've been doing some thinking and coding today on the dts-versioning / new-style literals issue. Here's a set of two patches which need a little polish, but I think demonstrate a better way to handle the transition; and a start on some not-directly-related improvements to the parser design. Some rationales:
- As I've been saying, I think *any* passing of data from the
parser back to the lexer to control its behaviour is asking for (at the least) very confusing flow of control.
- property/node names suck. To cover the various existing
things in Apple/IBM firmwares, they have to allow all sorts of nasty punctuation (e.g. +*?-) which could cause great trouble with expressions later on.
- therefore, I think I was thinking arse-backwards when I came
up with the lexer states for CELLDATA and MEMRESERVE. Instead recognizing sensible literals and sensible identifiers should be the normal lexer behaviour, and recognizing the weird the prop/node names should be the special lexer state.
- I was always a bit dubious about the very-missible visual
difference between
/memreserve/ 0x80000 0xf0000; and /memreserve/ 0x80000-0xf0000;
Plus the "range" form doesn't seem to have been used by any in-kernel dts files. Therefore, rather than changing the range symbol, simply drop support for the range form in the new version.
- Rather than making lexer rules be narrow, so that only valid
symbols will be lexed, it's actually better to make the lexer rules broad (not conflicting with other valid things, obviously), then verify that the returned tokens are really valid further up the stack. This way we can generate a meaningful "bad symbol/identifier/name/literal" message, instead of simply getting "syntax error" - or worse, splitting the bogus token into multiple valid tokens which we manage to parse far enough to get a thoroughly confusing state. (Imagine if a mistype like <0xdeadgeef>, parsed as a 2-cell value, <0xdead> then <geef>, with geef assumed to be a variable/identifier. Or if <077877> was parsed as octal 077 then 877 decimal.)
Some things that I know need polish in these patches:
- Currently if we get badly formatted literals, we print an
error message with yyerror(), but parsing continues using I'm not sure quite what crazy assumed value for the literal.
- -Odts output is still v0.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 1/2 dtc: Simplify lexing/parsing of literals vs. node/property names
The current scheme of having CELLDATA and MEMRESERVE states to recognize hex literals instead of node or property names is arse-backwards. The patch switches things around so that literals are lexed in normal states, and property/node names are only recognized in the special PROPNODENAME state, which is only entered after a { or a ;, and is left as soon as we scan a property/node name or a keyword.
--- dtc-lexer.l | 105 +++++++++++--------------------------------------- dtc-parser.y | 123 ++++++++++++++++++++++++++--------------------------------- 2 files changed, 80 insertions(+), 148 deletions(-)
dtc-lexer.l +25 -80
@@ -21,9 +21,8 @@ %option noyywrap nounput yylineno %x INCLUDE -%x CELLDATA %x BYTESTRING -%x MEMRESERVE +%x PROPNODENAME PROPCHAR [a-zA-Z0-9,._+*#?-] UNITCHAR [0-9a-f,]
@@ -51,7 +50,7 @@ %% -"/include/" BEGIN(INCLUDE); +<*>"/include/" BEGIN(INCLUDE); <INCLUDE>\"[^"\n]*\" { yytext[strlen(yytext) - 1] = 0;
@@ -63,13 +62,13 @@ } -<<EOF>> { +<*><<EOF>> { if (!pop_input_file()) { yyterminate(); } } -\"([^\\"]|\\.)*\" { +<*>\"([^\\"]|\\.)*\" { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("String: %s\n", yytext);
@@ -79,45 +78,24 @@ return DT_STRING; } -"/memreserve/" { +<*>"/memreserve/" { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("Keyword: /memreserve/\n"); - BEGIN(MEMRESERVE); - return DT_MEMRESERVE; - } - -<MEMRESERVE>[0-9a-fA-F]+ { - yylloc.filenum = srcpos_filenum; - yylloc.first_line = yylineno; - if (yyleng > 2*sizeof(yylval.addr)) { - fprintf(stderr, "Address value %s too large\n", - yytext); - } - yylval.addr = (u64) strtoull(yytext, NULL, 16); - DPRINT("Addr: %llx\n", - (unsigned long long)yylval.addr); - return DT_ADDR; - } - -<MEMRESERVE>";" { - yylloc.filenum = srcpos_filenum; - yylloc.first_line = yylineno; - DPRINT("/MEMRESERVE\n"); BEGIN(INITIAL); - return ';'; + return DT_MEMRESERVE; } <*>[a-zA-Z_][a-zA-Z0-9_]*: { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("Label: %s\n", yytext); - yylval.str = strdup(yytext); - yylval.str[yyleng-1] = '\0'; + yylval.labelref = strdup(yytext); + yylval.labelref[yyleng-1] = '\0'; return DT_LABEL; } -<CELLDATA>[bodh]# { +[bodh]# { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; if (*yytext == 'b')
@@ -132,27 +110,19 @@ return DT_BASE; } -<CELLDATA>[0-9a-fA-F]+ { - yylloc.filenum = srcpos_filenum; - yylloc.first_line = yylineno; - yylval.str = strdup(yytext); - DPRINT("Cell: '%s'\n", yylval.str); - return DT_CELL; - } - -<CELLDATA>">" { +[0-9a-fA-F]+ { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; - DPRINT("/CELLDATA\n"); - BEGIN(INITIAL); - return '>'; + yylval.literal = strdup(yytext); + DPRINT("Literal: '%s'\n", yylval.literal); + return DT_LITERAL; } -<CELLDATA>\&{REFCHAR}* { +\&{REFCHAR}* { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("Ref: %s\n", yytext+1); - yylval.str = strdup(yytext+1); + yylval.labelref = strdup(yytext+1); return DT_REF; }
@@ -172,30 +142,13 @@ return ']'; } -, { /* Technically this is a valid property name, - but we'd rather use it as punctuation, so detect it - here in preference */ - yylloc.filenum = srcpos_filenum; - yylloc.first_line = yylineno; - DPRINT("Char (propname like): %c (\\x%02x)\n", yytext[0], - (unsigned)yytext[0]); - return yytext[0]; - } - -{PROPCHAR}+ { +<PROPNODENAME>{PROPCHAR}+(@{UNITCHAR}+)? { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; - DPRINT("PropName: %s\n", yytext); - yylval.str = strdup(yytext); - return DT_PROPNAME; - } - -{PROPCHAR}+(@{UNITCHAR}+)? { - yylloc.filenum = srcpos_filenum; - yylloc.first_line = yylineno; - DPRINT("NodeName: %s\n", yytext); - yylval.str = strdup(yytext); - return DT_NODENAME; + DPRINT("PropNodeName: %s\n", yytext); + yylval.propnodename = strdup(yytext); + BEGIN(INITIAL); + return DT_PROPNODENAME; }
@@ -213,21 +166,13 @@ <*>. { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; - switch (yytext[0]) { - case '<': - DPRINT("CELLDATA\n"); - BEGIN(CELLDATA); - break; - case '[': - DPRINT("BYTESTRING\n"); - BEGIN(BYTESTRING); - break; - default: - + if ((yytext[0] == '{') + || (yytext[0] == ';')) { + DPRINT("<PROPNODENAME>\n"); + BEGIN(PROPNODENAME); + } DPRINT("Char: %c (\\x%02x)\n", yytext[0], (unsigned)yytext[0]); - break; - } return yytext[0]; }
dtc-parser.y +55 -68
220 lines not on this page · raw message
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 2/2 dtc: Switch dtc to C-style literals
In reply to David GibsonThis patch introduces a new version of dts file, distinguished from older files by starting with the special token /dts-v1/. dts files in the new version take C-style literals instead of the old bare hex or OF-style base notation. In addition, the "range" for of memreserve entries (/memreserve/ f0000-fffff) is no longer recognized in the new format.
--- dtc-lexer.l | 40 ++++++++++++++++++++++++++++++++-------- dtc-parser.y | 38 ++++++++++++++++++++++++++++++++++++-- tests/run_tests.sh | 4 ++++ tests/test_tree1.dts | 16 +++++++++------- tests/test_tree1_dts0.dts | 27 +++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 17 deletions(-)
dtc-lexer.l +32 -8
@@ -23,6 +23,7 @@ %x INCLUDE %x BYTESTRING %x PROPNODENAME +%s V1 PROPCHAR [a-zA-Z0-9,._+*#?-] UNITCHAR [0-9a-f,]
@@ -44,12 +45,18 @@ #define DPRINT(fmt, ...) do { } while (0) #endif +static int dts_version; /* = 0 */ - +#define BEGIN_DEFAULT() if (dts_version == 0) { \ + DPRINT("<INITIAL>\n"); \ + BEGIN(INITIAL); \ + } else { \ + DPRINT("<V1>\n"); \ + BEGIN(V1); \ + } %} %% - <*>"/include/" BEGIN(INCLUDE); <INCLUDE>\"[^"\n]*\" {
@@ -58,7 +65,7 @@ /* Some unrecoverable error.*/ exit(1); } - BEGIN(INITIAL); + BEGIN_DEFAULT(); }
@@ -78,11 +85,20 @@ return DT_STRING; } +<*>"/dts-v1/" { + yylloc.filenum = srcpos_filenum; + yylloc.first_line = yylineno; + DPRINT("Keyword: /dts-v1/\n"); + dts_version = 1; + BEGIN_DEFAULT(); + return DT_V1; + } + <*>"/memreserve/" { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("Keyword: /memreserve/\n"); - BEGIN(INITIAL); + BEGIN_DEFAULT(); return DT_MEMRESERVE; }
@@ -95,7 +111,7 @@ return DT_LABEL; } -[bodh]# { +<INITIAL>[bodh]# { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; if (*yytext == 'b')
@@ -110,7 +126,15 @@ return DT_BASE; } -[0-9a-fA-F]+ { +<INITIAL>[0-9a-fA-F]+ { + yylloc.filenum = srcpos_filenum; + yylloc.first_line = yylineno; + yylval.literal = strdup(yytext); + DPRINT("Literal: '%s'\n", yylval.literal); + return DT_LEGACYLITERAL; + } + +<V1>[0-9]+|0[xX][0-9a-fA-F]+ { yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; yylval.literal = strdup(yytext);
@@ -138,7 +162,7 @@ yylloc.filenum = srcpos_filenum; yylloc.first_line = yylineno; DPRINT("/BYTESTRING\n"); - BEGIN(INITIAL); + BEGIN_DEFAULT(); return ']'; }
@@ -147,7 +171,7 @@ yylloc.first_line = yylineno; DPRINT("PropNodeName: %s\n", yytext); yylval.propnodename = strdup(yytext); - BEGIN(INITIAL); + BEGIN_DEFAULT(); return DT_PROPNODENAME; }
dtc-parser.y +36 -2
@@ -48,9 +48,11 @@ struct reserve_info *re; } +%token DT_V1 %token DT_MEMRESERVE %token <propnodename> DT_PROPNODENAME %token <literal> DT_LITERAL +%token <literal> DT_LEGACYLITERAL %token <cbase> DT_BASE %token <byte> DT_BYTE %token <data> DT_STRING
@@ -61,6 +63,8 @@ %type <data> propdataprefix %type <re> memreserve %type <re> memreserves +%type <re> v0_memreserve +%type <re> v0_memreserves %type <addr> addr %type <data> celllist %type <cbase> cellbase
@@ -78,7 +82,11 @@ %% sourcefile: - memreserves devicetree + DT_V1 ';' memreserves devicetree + { + the_boot_info = build_boot_info($3, $4); + } + | v0_memreserves devicetree { the_boot_info = build_boot_info($1, $2); }
@@ -100,6 +108,24 @@ { $$ = build_reserve_entry($3, $4, $1); } + ; + +v0_memreserves: + /* empty */ + { + $$ = NULL; + } + | v0_memreserve v0_memreserves + { + $$ = chain_reserve_entry($1, $2); + }; + ; + +v0_memreserve: + memreserve + { + $$ = $1; + } | label DT_MEMRESERVE addr '-' addr ';' { $$ = build_reserve_entry($3, $5 - $3 + 1, $1);
@@ -108,6 +134,10 @@ addr: DT_LITERAL + { + $$ = eval_literal($1, 0, 64); + } + | DT_LEGACYLITERAL { $$ = eval_literal($1, 16, 64); }
@@ -212,7 +242,11 @@ ; cellval: - cellbase DT_LITERAL + DT_LITERAL + { + $$ = eval_literal($1, 0, 32); + } + | cellbase DT_LEGACYLITERAL { $$ = eval_literal($2, $1, 32); }
tests/run_tests.sh +4 -0
@@ -118,6 +118,10 @@ tree1_tests_rw dtc_tree1.test.dtb run_test dtbs_equal_ordered dtc_tree1.test.dtb test_tree1.dtb + run_test dtc.sh -I dts -O dtb -o dtc_tree1_dts0.test.dtb test_tree1_dts0.dts + tree1_tests dtc_tree1_dts0.test.dtb + tree1_tests_rw dtc_tree1_dts0.test.dtb + run_test dtc.sh -I dts -O dtb -o dtc_escapes.test.dtb escapes.dts run_test string_escapes dtc_escapes.test.dtb }
tests/test_tree1.dts +9 -7
@@ -1,27 +1,29 @@ -/memreserve/ deadbeef00000000-deadbeef000fffff; -/memreserve/ abcd1234 00001234; +/dts-v1/; + +/memreserve/ 0xdeadbeef00000000 0x100000; +/memreserve/ 0xabcd1234 0x00001234; / { compatible = "test_tree1"; - prop-int = <deadbeef>; + prop-int = <0xdeadbeef>; prop-str = "hello world"; subnode@1 { compatible = "subnode1"; - prop-int = <deadbeef>; + prop-int = <0xdeadbeef>; subsubnode { compatible = "subsubnode1", "subsubnode"; - prop-int = <deadbeef>; + prop-int = <0xdeadbeef>; }; }; subnode@2 { - prop-int = <abcd1234>; + prop-int = <0xabcd1234>; subsubnode@0 { compatible = "subsubnode2", "subsubnode"; - prop-int = <abcd1234>; + prop-int = <0xabcd1234>; }; }; };
tests/test_tree1_dts0.dts new file +27 -0
@@ -0,0 +1,27 @@ +/memreserve/ deadbeef00000000-deadbeef000fffff; +/memreserve/ abcd1234 00001234; + +/ { + compatible = "test_tree1"; + prop-int = <deadbeef>; + prop-str = "hello world"; + + subnode@1 { + compatible = "subnode1"; + prop-int = <deadbeef>; + + subsubnode { + compatible = "subsubnode1", "subsubnode"; + prop-int = <deadbeef>; + }; + }; + + subnode@2 { + prop-int = <abcd1234>; + + subsubnode@0 { + compatible = "subsubnode2", "subsubnode"; + prop-int = <abcd1234>; + }; + }; +};
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson