Input file and srcpos rework (resend again)
cover Input file and srcpos rework (resend again)
Thought I'd throw this patch series out there again. It never got merge on the previous occasions I posted it because it conflicts with jdl's language enhancement patches.
But the language extension stuff seems to have fallen by the wayside for the time being, and in any case it shouldn't be that hard to update the langauge stuff on top of these cleanups.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 1/4 dtc: Simpler interface to source file management
This patch cleans up our handling of input files, particularly dts source files, but also (to an extent) other input files such as those used by /incbin/ and those used in -I dtb and -I fs modes.
We eliminate the current clunky mechanism which combines search paths (which we don't actually use at present) with the open relative to current source file behaviour, which we do.
Instead there's a single srcfile_relative_open() entry point for callers which opens a new input file relative to the current source file (which the srcpos code tracks internally). It doesn't currently do search paths, but we can add that later without messing with the callers, by drawing the search path from a global (which makes sense anyway, rather than shuffling it around the rest of the processing code).
That suffices for non-dts input files. For the actual dts files, srcfile_push() and srcfile_pop() wrappers open the file while also keeping track of it as the current source file for future opens.
--- convert-dtsv0-lexer.l | 6 + dtc-lexer.l | 91 +++--------------------------- dtc-parser.y | 18 ++--- dtc.c | 24 ------- dtc.h | 4 - flattree.c | 24 +++---- srcpos.c | 151 ++++++++++++++++++++++---------------------------- srcpos.h | 27 +++----- treesource.c | 4 - util.c | 24 +++++++ util.h | 1 11 files changed, 141 insertions(+), 233 deletions(-)
dtc.c +0 -24
@@ -32,30 +32,6 @@ int minsize; /* Minimum blob size */int padsize;/* Additional padding to blob */ int phandle_format =PHANDLE_BOTH ;/* Use linux,phandle or phandle properties */ -char *join_path (const char *path,const char *name) -{ -int lenp =strlen (path); -int lenn =strlen (name); -int len; -int needslash =1 ; -char *str; - - len = lenp + lenn +2 ; -if ((lenp >0 ) && (path[lenp-1 ] =='/' )) { - needslash =0 ; - len--; - } - - str =xmalloc (len); -memcpy (str, path, lenp); -if (needslash) { - str[lenp] ='/' ; - lenp++; - } -memcpy (str+lenp, name, lenn+1 ); -return str; -} -static void fill_fullpaths (struct node *tree,const char *prefix) {struct node *child;
convert-dtsv0-lexer.l +4 -2
@@ -210,8 +210,10 @@ static void convert_file(const char *fna memcpy(newname, fname, len); memcpy(newname + len, suffix, sizeof(suffix)); - srcpos_file = dtc_open_file(fname, NULL); - yyin = srcpos_file->file; + yyin = fopen(fname, "r"); + if (!yyin) + die("Couldn't open input file %s: %s\n", + fname, strerror(errno)); yyout = fopen(newname, "w"); if (!yyout)
flattree.c +12 -12
@@ -797,7 +797,7 @@ static struct node *unflatten_tree(strucstruct boot_info *dt_from_blob (const char *fname) { -struct dtc_file *dtcf; +FILE *f;uint32_t magic, totalsize, version, size_dt, boot_cpuid_phys;uint32_t off_dt, off_str, off_mem_rsvmap;int rc;
@@ -812,14 +812,14 @@ struct boot_info *dt_from_blob(const chauint32_t val;int flags =0 ; - dtcf =dtc_open_file (fname,NULL ); + f =srcfile_relative_open (fname,NULL ); - rc =fread (&magic,sizeof (magic),1 , dtcf->file); -if (ferror (dtcf->file)) + rc =fread (&magic,sizeof (magic),1 , f); +if (ferror (f))die ("Error reading DT blob magic number: %s\n" ,strerror (errno));if (rc <1 ) { -if (feof (dtcf->file)) +if (feof (f))die ("EOF reading DT blob magic number\n" );else die ("Mysterious short read reading magic number\n" );
@@ -829,11 +829,11 @@ struct boot_info *dt_from_blob(const chaif (magic !=FDT_MAGIC )die ("Blob has incorrect magic number\n" ); - rc =fread (&totalsize,sizeof (totalsize),1 , dtcf->file); -if (ferror (dtcf->file)) + rc =fread (&totalsize,sizeof (totalsize),1 , f); +if (ferror (f))die ("Error reading DT blob size: %s\n" ,strerror (errno));if (rc <1 ) { -if (feof (dtcf->file)) +if (feof (f))die ("EOF reading DT blob size\n" );else die ("Mysterious short read reading blob size\n" );
@@ -853,12 +853,12 @@ struct boot_info *dt_from_blob(const cha p = blob +sizeof (magic) +sizeof (totalsize);while (sizeleft) { -if (feof (dtcf->file)) +if (feof (f))die ("EOF before reading %d bytes of DT blob\n" , totalsize); - rc =fread (p,1 , sizeleft, dtcf->file); -if (ferror (dtcf->file)) + rc =fread (p,1 , sizeleft, f); +if (ferror (f))die ("Error reading DT blob: %s\n" ,strerror (errno));
@@ -921,7 +921,7 @@ struct boot_info *dt_from_blob(const chafree (blob); -dtc_close_file (dtcf); +fclose (f);return build_boot_info (reservelist, tree, boot_cpuid_phys); }
treesource.c +2 -2
@@ -32,8 +32,8 @@ struct boot_info *dt_from_source(const c the_boot_info =NULL ; treesource_error =0 ; - srcpos_file =dtc_open_file (fname,NULL ); - yyin = srcpos_file->file; +srcfile_push (fname); + yyin = current_srcfile->f;if (yyparse () !=0 )die ("Unable to parse input tree\n" );
dtc-lexer.l +10 -81
117 lines not on this page · raw message
dtc-parser.y +8 -10
@@ -169,33 +169,31 @@ propdata: } | propdataprefix DT_INCBIN '(' DT_STRING ',' addr ',' addr ')' { - struct search_path path = { srcpos_file->dir, NULL, NULL }; - struct dtc_file *file = dtc_open_file($4.val, &path); - struct data d = empty_data; + FILE *f = srcfile_relative_open($4.val, NULL); + struct data d; if ($6 != 0) - if (fseek(file->file, $6, SEEK_SET) != 0) + if (fseek(f, $6, SEEK_SET) != 0) srcpos_error(&yylloc, "Couldn't seek to offset %llu in \"%s\": %s", (unsigned long long)$6, $4.val, strerror(errno)); - d = data_copy_file(file->file, $8); + d = data_copy_file(f, $8); $$ = data_merge($1, d); - dtc_close_file(file); + fclose(f); } | propdataprefix DT_INCBIN '(' DT_STRING ')' { - struct search_path path = { srcpos_file->dir, NULL, NULL }; - struct dtc_file *file = dtc_open_file($4.val, &path); + FILE *f = srcfile_relative_open($4.val, NULL); struct data d = empty_data; - d = data_copy_file(file->file, -1); + d = data_copy_file(f, -1); $$ = data_merge($1, d); - dtc_close_file(file); + fclose(f); } | propdata DT_LABEL {
dtc.h +0 -4
@@ -224,8 +224,4 @@ struct boot_info *dt_from_source(const cstruct boot_info *dt_from_fs (const char *dirname); -/* misc */ - -char *join_path (const char *path,const char *name); -#endif /* _DTC_H */
util.c +24 -0
@@ -28,3 +28,27 @@ char *xstrdup(const char *s)return dup; } + +char *join_path (const char *path,const char *name) +{ +int lenp =strlen (path); +int lenn =strlen (name); +int len; +int needslash =1 ; +char *str; + + len = lenp + lenn +2 ; +if ((lenp >0 ) && (path[lenp-1 ] =='/' )) { + needslash =0 ; + len--; + } + + str =xmalloc (len); +memcpy (str, path, lenp); +if (needslash) { + str[lenp] ='/' ; + lenp++; + } +memcpy (str+lenp, name, lenn+1 ); +return str; +}
util.h +1 -0
@@ -51,5 +51,6 @@ static inline void *xrealloc(void *p, si }extern char *xstrdup (const char *s); +extern char *join_path (const char *path,const char *name);#endif /* _UTIL_H */
srcpos.c +68 -83
185 lines not on this page · raw message
srcpos.h +12 -15
@@ -27,19 +27,27 @@#include <stdio.h> -struct dtc_file { +struct srcfile_state { +FILE *f; +char *name;char *dir; -const char *name; -FILE *file; +int lineno; +struct srcfile_state *prev; }; +extern struct srcfile_state *current_srcfile;/* = NULL */ + +FILE *srcfile_relative_open (const char *fname,char **fullnamep); +void srcfile_push (const char *fname); +int srcfile_pop (void ); +#if ! defined(YYLTYPE ) && ! defined(YYLTYPE_IS_DECLARED )typedef struct YYLTYPE {int first_line;int first_column;int last_line;int last_column; -struct dtc_file *file; +struct srcfile_state *file; }YYLTYPE ;#define YYLTYPE_IS_DECLARED 1
@@ -81,17 +89,6 @@ typedef YYLTYPE srcpos;*/ extern srcpos srcpos_empty; -extern struct dtc_file *srcpos_file; - -struct search_path { -const char *dir;/* NULL for current directory */ -struct search_path *prev, *next; -}; - -extern struct dtc_file *dtc_open_file (const char *fname, -const struct search_path *search); -extern void dtc_close_file (struct dtc_file *file); -extern srcpos *srcpos_copy (srcpos *pos);extern char *srcpos_string (srcpos *pos);extern void srcpos_dump (srcpos *pos);
PATCH 2/4 dtc: Cleanup line number tracking, add column number tracking
In reply to David GibsonOur YYLTYPE current carries around first and last line and first and last column information. However, of these, on the first line information is actually filled in properly.
Furthermore, filling in the line number information from yylineno is kind of clunky: we have to copy its value to the srcfile stack and back to handle include file positioning correctly.
This patch cleans this up. We turn off flex's yylineno option and instead track the line and column number ourselves from YY_USER_ACTION. The line and column number are stored directly inside the srcfile_state structure, so it's automatically a per-file quantity. We now also fill in all the yylloc from YY_USER_ACTION.
--- dtc-lexer.l | 11 ++++------- srcpos.c | 33 ++++++++++++++++++++++++++++++++- srcpos.h | 3 ++- 3 files changed, 38 insertions(+), 9 deletions(-)
srcpos.c +32 -1
@@ -87,6 +87,10 @@ void srcfile_push(const char *fname) srcfile->f =srcfile_relative_open (fname, &srcfile->name); srcfile->dir =dirname (srcfile->name); srcfile->prev = current_srcfile; + + srcfile->lineno =1 ; + srcfile->colno =1 ; + current_srcfile = srcfile; }
@@ -99,7 +103,8 @@ int srcfile_pop(void) current_srcfile = srcfile->prev;if (fclose (srcfile->f)) -die ("Error closing \"%s\": %s\n" , srcfile->name,strerror (errno)); +die ("Error closing \"%s\": %s\n" , srcfile->name, +strerror (errno));/* FIXME: We allow the srcfile_state structure to leak, * because it could still be referenced from a location
@@ -122,6 +127,32 @@ srcpos srcpos_empty = { .file = NULL, }; +#define TAB_SIZE 8 + +void srcpos_update (srcpos *pos,const char *text,int len) +{ +int i; + + pos->file = current_srcfile; + + pos->first_line = current_srcfile->lineno; + pos->first_column = current_srcfile->colno; + +for (i =0 ; i < len; i++) +if (text[i] =='\n' ) { + current_srcfile->lineno++; + current_srcfile->colno =1 ; + }else if (text[i] =='\t' ) { + current_srcfile->colno = +ALIGN (current_srcfile->colno,TAB_SIZE ); + }else { + current_srcfile->colno++; + } + + pos->last_line = current_srcfile->lineno; + pos->last_column = current_srcfile->colno; +} +srcpos *srcpos_copy (srcpos *pos) {
dtc-lexer.l +4 -7
@@ -18,7 +18,7 @@ * USA */ -%option noyywrap nounput noinput yylineno +%option noyywrap nounput noinput %x INCLUDE %x BYTESTRING
@@ -38,10 +38,11 @@ LINECOMMENT "//".*\n #include "srcpos.h" #include "dtc-parser.tab.h" + +/* CAUTION: this will stop working if we ever use yyless() or yyunput() */ #define YY_USER_ACTION \ { \ - yylloc.file = current_srcfile; \ - yylloc.first_line = yylineno; \ + srcpos_update(&yylloc, yytext, yyleng); \ } /*#define LEXDEBUG 1*/
@@ -169,12 +170,9 @@ static void push_input_file(const char * { assert(filename); - current_srcfile->lineno = yylineno; - srcfile_push(filename); yyin = current_srcfile->f; - yylineno = 1; yypush_buffer_state(yy_create_buffer(yyin, YY_BUF_SIZE)); }
@@ -186,7 +184,6 @@ static int pop_input_file(void) return 0; yypop_buffer_state(); - yylineno = current_srcfile->lineno; yyin = current_srcfile->f; return 1;
srcpos.h +2 -1
@@ -31,7 +31,7 @@ struct srcfile_state {FILE *f;char *name;char *dir; -int lineno; +int lineno, colno;struct srcfile_state *prev; };
@@ -89,6 +89,7 @@ typedef YYLTYPE srcpos;*/ extern srcpos srcpos_empty; +extern void srcpos_update (srcpos *pos,const char *text,int len);extern srcpos *srcpos_copy (srcpos *pos);extern char *srcpos_string (srcpos *pos);extern void srcpos_dump (srcpos *pos);
PATCH 3/4 dtc: Cleanup srcpos_string()
In reply to David GibsonThere are several small problems with the current srcpos_string().
- The code unnecessarily uses a temp buffer and two rounds of *printf(); a single asprintf() will suffice.
- With previous changes, pos->file->name can never be NULL, and the name field for a srcfile bound to stdin is already set to something sensible.
- On allocation failure in asprintf() it returns a bogus result, instead of causing a fatal error like every other failed allocation.
- The format for representing file/line/column is gratuitously different from the file/line format we used to use, and the format used by gcc and bison.
This patch addresses all of these. There remains the problem that asprintf() is not portable, but that can wait until another patch.
--- srcpos.c | 49 ++++++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 31 deletions(-)
srcpos.c +18 -31
@@ -182,41 +182,28 @@ srcpos_dump(srcpos *pos)char *srcpos_string (srcpos *pos) { -const char *fname; -char col_buf[100 ]; +const char *fname ="<no-file>" ;char *pos_str; +int rc; -if (!pos) { - fname ="<no-file>" ; - }else if (pos->file->name) { +if (pos) fname = pos->file->name; -if (strcmp (fname,"-" ) ==0 ) - fname ="stdin" ; - }else { - fname ="<no-file>" ; - } - -if (pos->first_line == pos->last_line) { -if (pos->first_column == pos->last_column) { -snprintf (col_buf,sizeof (col_buf), -"%d:%d" , - pos->first_line, pos->first_column); - }else { -snprintf (col_buf,sizeof (col_buf), -"%d:%d-%d" , - pos->first_line, - pos->first_column, pos->last_column); - } - - }else { -snprintf (col_buf,sizeof (col_buf), -"%d:%d - %d:%d" , - pos->first_line, pos->first_column, - pos->last_line, pos->last_column); - } -if (asprintf (&pos_str,"%s %s" , fname, col_buf) ==-1 ) -return "<unknown source position?" ; + +if (pos->first_line != pos->last_line) + rc =asprintf (&pos_str,"%s:%d.%d-%d.%d" , fname, + pos->first_line, pos->first_column, + pos->last_line, pos->last_column); +else if (pos->first_column != pos->last_column) + rc =asprintf (&pos_str,"%s:%d.%d-%d" , fname, + pos->first_line, pos->first_column, + pos->last_column); +else + rc =asprintf (&pos_str,"%s:%d.%d" , fname, + pos->first_line, pos->first_column); + +if (rc ==-1 ) +die ("Couldn't allocate in srcpos string" );return pos_str; }
PATCH 4/4 dtc: Cleanup YYLTYPE and YYLLOC_DEFAULT declarations
In reply to David GibsonThis patch makes some small cleanups to the declaration of YYLTYPE, YYLLOC_DEFAULT and related things.
- We used to use undocumented magic #defines for bison, YYLTYPE_IS_DECLARED and YYLTYPE_IS_TRIVIAL. This may not be portable across bison versions. Instead define YYLTYPE as a macro in terms of struct srcpos, as the info pages suggest.
- Our kernel-derived coding style discourages typedefed structures. So use 'struct srcpos' instead of 'srcpos' throughout'.
- Indent the YYLLOC_DEFAULT macro according to our coding style (it was in GNU indent style, since it was taken from the example in the bison info).
--- srcpos.c | 22 +++++++++---------- srcpos.h | 70 +++++++++++++++++++++++---------------------------------------- 2 files changed, 37 insertions(+), 55 deletions(-)
srcpos.h +26 -44
@@ -20,11 +20,6 @@#ifndef _SRCPOS_H_#define _SRCPOS_H_ -/* -* Augment the standard YYLTYPE with a filenum index into an -* array of all opened filenames. -*/ -#include <stdio.h> struct srcfile_state {
@@ -41,62 +36,49 @@ FILE *srcfile_relative_open(const char *void srcfile_push (const char *fname);int srcfile_pop (void ); -#if ! defined(YYLTYPE ) && ! defined(YYLTYPE_IS_DECLARED ) -typedef struct YYLTYPE { +struct srcpos {int first_line;int first_column;int last_line;int last_column;struct srcfile_state *file; -}YYLTYPE ; +}; -#define YYLTYPE_IS_DECLARED 1 -#define YYLTYPE_IS_TRIVIAL 1 -#endif - -/* Cater to old parser templates. */ -#ifndef YYID -#define YYID (n) (n) -#endif - -#define YYLLOC_DEFAULT (Current, Rhs,N ) \ - do \ - if (YYID (N)) \ - { \ - (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \ - (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \ - (Current).last_line = YYRHSLOC (Rhs, N).last_line; \ - (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ - (Current).file = YYRHSLOC (Rhs, N).file; \ - } \ - else \ - { \ - (Current).first_line = (Current).last_line = \ - YYRHSLOC (Rhs, 0).last_line; \ - (Current).first_column = (Current).last_column = \ - YYRHSLOC (Rhs, 0).last_column; \ - (Current).file = YYRHSLOC (Rhs, 0).file; \ - } \ - while (YYID (0)) +#define YYLTYPE struct srcpos +#define YYLLOC_DEFAULT (Current, Rhs,N ) \ + do { \ + if (N) { \ + (Current).first_line = YYRHSLOC(Rhs, 1).first_line; \ + (Current).first_column = YYRHSLOC(Rhs, 1).first_column; \ + (Current).last_line = YYRHSLOC(Rhs, N).last_line; \ + (Current).last_column = YYRHSLOC (Rhs, N).last_column; \ + (Current).file = YYRHSLOC(Rhs, N).file; \ + } else { \ + (Current).first_line = (Current).last_line = \ + YYRHSLOC(Rhs, 0).last_line; \ + (Current).first_column = (Current).last_column = \ + YYRHSLOC(Rhs, 0).last_column; \ + (Current).file = YYRHSLOC (Rhs, 0).file; \ + } \ + } while (0) -typedef YYLTYPE srcpos ;/* * Fictional source position used for IR nodes that are * created without otherwise knowing a true source position. * For example,constant definitions from the command line. */ -extern srcpos srcpos_empty; +extern struct srcpos srcpos_empty; -extern void srcpos_update (srcpos *pos,const char *text,int len); -extern srcpos *srcpos_copy (srcpos *pos); -extern char *srcpos_string (srcpos *pos); -extern void srcpos_dump (srcpos *pos); +extern void srcpos_update (struct srcpos *pos,const char *text,int len); +extern struct srcpos *srcpos_copy (struct srcpos *pos); +extern char *srcpos_string (struct srcpos *pos); +extern void srcpos_dump (struct srcpos *pos); -extern void srcpos_error (srcpos *pos,char const *, ...) +extern void srcpos_error (struct srcpos *pos,char const *, ...) __attribute__((format (printf,2 ,3 ))); -extern void srcpos_warn (srcpos *pos,char const *, ...) +extern void srcpos_warn (struct srcpos *pos,char const *, ...) __attribute__((format (printf,2 ,3 )));#endif /* _SRCPOS_H_ */
srcpos.c +11 -11
@@ -119,7 +119,7 @@ int srcfile_pop(void)* The empty source position. */ -srcpos srcpos_empty = { +struct srcpos srcpos_empty = { .first_line =0 , .first_column =0 , .last_line =0 ,
@@ -129,7 +129,7 @@ srcpos srcpos_empty = {#define TAB_SIZE 8 -void srcpos_update (srcpos *pos,const char *text,int len) +void srcpos_update (struct srcpos *pos,const char *text,int len) {int i;
@@ -153,13 +153,13 @@ void srcpos_update(srcpos *pos, const ch pos->last_column = current_srcfile->colno; } -srcpos * -srcpos_copy (srcpos *pos) +struct srcpos * +srcpos_copy (struct srcpos *pos) { -srcpos *pos_new; +struct srcpos *pos_new; - pos_new =xmalloc (sizeof (srcpos)); -memcpy (pos_new, pos,sizeof (srcpos)); + pos_new =xmalloc (sizeof (struct srcpos )); +memcpy (pos_new, pos,sizeof (struct srcpos ));return pos_new; }
@@ -167,7 +167,7 @@ srcpos_copy(srcpos *pos)void -srcpos_dump (srcpos *pos) +srcpos_dump (struct srcpos *pos) {printf ("file : \"%s\"\n" , pos->file ? (char *) pos->file :"<no file>" );
@@ -180,7 +180,7 @@ srcpos_dump(srcpos *pos)char * -srcpos_string (srcpos *pos) +srcpos_string (struct srcpos *pos) {const char *fname ="<no-file>" ;char *pos_str;
@@ -210,7 +210,7 @@ srcpos_string(srcpos *pos)void -srcpos_error (srcpos *pos,char const *fmt, ...) +srcpos_error (struct srcpos *pos,char const *fmt, ...) {const char *srcstr;va_list va;
@@ -227,7 +227,7 @@ srcpos_error(srcpos *pos, char const *fmvoid -srcpos_warn (srcpos *pos,char const *fmt, ...) +srcpos_warn (struct srcpos *pos,char const *fmt, ...) {const char *srcstr;va_list va;
Quoted 7 lines
Thought I'd throw this patch series out there again. It never got merge on the previous occasions I posted it because it conflicts with jdl's language enhancement patches.
But the language extension stuff seems to have fallen by the wayside for the time being, and in any case it shouldn't be that hard to update the langauge stuff on top of these cleanups.
All 4 of these patches have been applied and pushed out!
jdl