Eliminate the use of Autogen
cover Eliminate the use of Autogen
I already mentioned this on IRC, but I thought it'd be helpful to send it here as well.
This patch series (also in the cjwatson/new-autogen git branch) eliminates the use of Autogen and instead generates Automake input directly from gentpl.py.
Autogen's definitions files were useful, and I've maintained compatibility with the relevant subset of them, but the template language was cumbersome; beyond a certain level of complexity it became necessary to use the Guile extensions, and the impedance mismatch between those and the Python template generation was considerable. It's simpler, much faster, and very much less confusing to generate Makefile.*.am directly.
I've checked that this generates almost identical output, even when the modules from grub-extras are in use. The differences amount to a few unimportant things such as removal of trailing newlines, and the movement of a declarations block for libgnulib.a a bit further up the file; they are short enough to be easily reviewable and not significant enough to be worth contorting the generator code to avoid.
Colin Watson (3):
Generate Makefile.*.am directly from gentpl.py, eliminating the use of
Autogen. The Autogen definitions files remain intact as they offer a useful abstraction.
Show file name in error messages from AutogenParser. Handle #if/#endif and C-style comments in AutoGen definitions files.
.gitignore | 1 - ChangeLog | 6 + INSTALL | 1 - autogen.sh | 11 +- conf/Makefile.common | 15 +- conf/Makefile.extra-dist | 1 - gentpl.py | 1050 ++++++++++++++++++++++++++++------------------ 7 files changed, 651 insertions(+), 434 deletions(-)
Signature · 1 line
1.8.4.3
PATCH 1/3 Generate Makefile.*.am directly from gentpl.py, eliminating the use of Autogen. The Autogen definitions files remain intact as they offer a useful abstraction.
--- .gitignore | 1 - ChangeLog | 6 + INSTALL | 1 - autogen.sh | 11 +- conf/Makefile.common | 15 +- conf/Makefile.extra-dist | 1 - gentpl.py | 1007 +++++++++++++++++++++++++++------------------- 7 files changed, 608 insertions(+), 434 deletions(-)
.gitignore +0 -1
@@ -159,7 +159,6 @@ GPATH GRTAGS GSYMS GTAGS -Makefile.tpl compile depcomp mdate-sh
ChangeLog +6 -0
@@ -1,3 +1,9 @@ +2013-11-26 Colin Watson <[email protected]> + + Generate Makefile.*.am directly from gentpl.py, eliminating the use + of Autogen. The Autogen definitions files remain intact as they + offer a useful abstraction. + 2013-11-26 Vladimir Serbinenko <[email protected]> Add PCI command activation to all PCI drivers as required for coreboot
INSTALL +0 -1
@@ -55,7 +55,6 @@ need the following. * Python 2.5.2 or later * Autoconf 2.60 or later * Automake 1.10.1 or later -* Autogen 5.10 or later Prerequisites for make-check:
autogen.sh +3 -8
@@ -8,8 +8,6 @@ unset LC_ALL find . -iname '*.[ch]' ! -ipath './grub-core/lib/libgcrypt-grub/*' ! -ipath './build-aux/*' ! -ipath './grub-core/lib/libgcrypt/src/misc.c' ! -ipath './grub-core/lib/libgcrypt/src/global.c' ! -ipath './grub-core/lib/libgcrypt/src/secmem.c' ! -ipath './util/grub-gen-widthspec.c' ! -ipath './util/grub-gen-asciih.c' |sort > po/POTFILES.in find util -iname '*.in' ! -name Makefile.in |sort > po/POTFILES-shell.in -autogen --version >/dev/null || exit 1 - echo "Importing unicode..." python util/import_unicode.py unicode/UnicodeData.txt unicode/BidiMirroring.txt unicode/ArabicShaping.txt grub-core/unidata.c
@@ -32,10 +30,7 @@ for x in mpi-asm-defs.h mpih-add1.c mpih-sub1.c mpih-mul1.c mpih-mul2.c mpih-mul ln -s generic/"$x" grub-core/lib/libgcrypt-grub/mpi/"$x" done -echo "Creating Makefile.tpl..." -python gentpl.py | sed -e '/^$/{N;/^\n$/D;}' > Makefile.tpl - -echo "Running autogen..." +echo "Generating Automake input..." # Automake doesn't like including files from a path outside the project. rm -f contrib grub-core/contrib
@@ -59,8 +54,8 @@ for extra in contrib/*/Makefile.core.def; do fi done -cat $UTIL_DEFS | autogen -T Makefile.tpl | sed -e '/^$/{N;/^\n$/D;}' > Makefile.util.am -cat $CORE_DEFS | autogen -T Makefile.tpl | sed -e '/^$/{N;/^\n$/D;}' > grub-core/Makefile.core.am +python gentpl.py $UTIL_DEFS > Makefile.util.am +python gentpl.py $CORE_DEFS > grub-core/Makefile.core.am for extra in contrib/*/Makefile.common; do if test -e "$extra"; then
conf/Makefile.common +5 -10
@@ -131,20 +131,15 @@ EXTRA_DIST = CLEANFILES = BUILT_SOURCES = -# Rules for autogen definition files - -.PRECIOUS: $(top_srcdir)/Makefile.tpl -$(top_srcdir)/Makefile.tpl: $(top_srcdir)/gentpl.py - python $< | sed -e '/^$$/{N;/^\\n$$/D;}' > [email protected] || (rm -f [email protected]; exit 1) - mv [email protected] $@ +# Rules for Automake input .PRECIOUS: $(top_srcdir)/Makefile.util.am -$(top_srcdir)/Makefile.util.am: $(top_srcdir)/Makefile.util.def $(top_srcdir)/Makefile.utilgcry.def $(top_srcdir)/Makefile.tpl - cat $(top_srcdir)/Makefile.util.def $(top_srcdir)/Makefile.utilgcry.def | autogen -T $(top_srcdir)/Makefile.tpl | sed -e '/^$$/{N;/^\\n$$/D;}' > [email protected] || (rm -f [email protected]; exit 1) +$(top_srcdir)/Makefile.util.am: $(top_srcdir)/gentpl.py $(top_srcdir)/Makefile.util.def $(top_srcdir)/Makefile.utilgcry.def + python $^ > [email protected] || (rm -f [email protected]; exit 1) mv [email protected] $@ .PRECIOUS: $(top_srcdir)/grub-core/Makefile.core.am -$(top_srcdir)/grub-core/Makefile.core.am: $(top_srcdir)/grub-core/Makefile.core.def $(top_srcdir)/grub-core/Makefile.gcry.def $(top_srcdir)/Makefile.tpl +$(top_srcdir)/grub-core/Makefile.core.am: $(top_srcdir)/gentpl.py $(top_srcdir)/grub-core/Makefile.core.def $(top_srcdir)/grub-core/Makefile.gcry.def if [ "x$$GRUB_CONTRIB" != x ]; then echo "You need to run ./autogen.sh manually." >&2; exit 1; fi - cat $(top_srcdir)/grub-core/Makefile.core.def $(top_srcdir)/grub-core/Makefile.gcry.def | autogen -T $(top_srcdir)/Makefile.tpl | sed -e '/^$$/{N;/^\\n$$/D;}' > [email protected] || (rm -f [email protected]; exit 1) + python $^ > [email protected] || (rm -f [email protected]; exit 1) mv [email protected] $@
conf/Makefile.extra-dist +0 -1
@@ -2,7 +2,6 @@ EXTRA_DIST += autogen.sh EXTRA_DIST += geninit.sh EXTRA_DIST += gentpl.py -EXTRA_DIST += Makefile.tpl EXTRA_DIST += Makefile.util.def EXTRA_DIST += Makefile.utilgcry.def
gentpl.py +594 -413
1,120 lines not on this page · raw message
Signature · 1 line
1.8.4.3
PATCH 2/3 Show file name in error messages from AutogenParser.
In reply to Colin Watson--- gentpl.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-)
gentpl.py +21 -16
@@ -119,14 +119,15 @@ class AutogenState: need_idx, need_rbracket, indx_name, have_value, done) = range(12) class AutogenParseError(Exception): - def __init__(self, message, line): + def __init__(self, message, path, line): super(AutogenParseError, self).__init__(message) + self.path = path self.line = line def __str__(self): return ( super(AutogenParseError, self).__str__() + - " at line %d" % self.line) + " at file %s line %d" % (self.path, self.line)) class AutogenDefinition(list): def __getitem__(self, key):
@@ -161,6 +162,7 @@ class AutogenParser: self.def_stack = [("", self.definitions)] self.curdef = None self.new_name = None + self.cur_path = None self.cur_line = 0 @staticmethod
@@ -172,6 +174,9 @@ class AutogenParser: def is_value_name_char(c): return c in ":^-_" or c.isalnum() + def error(self, message): + raise AutogenParseError(message, self.cur_file, self.cur_line) + def read_tokens(self, f): data = f.read() end = len(data)
@@ -210,15 +215,13 @@ class AutogenParser: while True: offset += 1 if offset >= end: - raise AutogenParseError( - "EOF in quoted string", self.cur_line) + self.error("EOF in quoted string") if data[offset] == "\n": self.cur_line += 1 if data[offset] == "\\": offset += 1 if offset >= end: - raise AutogenParseError( - "EOF in quoted string", self.cur_line) + self.error("EOF in quoted string") if data[offset] == "\n": self.cur_line += 1 # Proper escaping unimplemented; this can be filled
@@ -260,21 +263,19 @@ class AutogenParser: yield AutogenToken.var_name, s offset = end_name else: - raise AutogenParseError( - "Invalid input character '%s'" % c, self.cur_line) + self.error("Invalid input character '%s'" % c) yield AutogenToken.eof, None def do_need_name_end(self, token): if len(self.def_stack) > 1: - raise AutogenParseError( - "Definition blocks were left open", self.cur_line) + self.error("Definition blocks were left open") def do_need_name_var_name(self, token): self.new_name = token def do_end_block(self, token): if len(self.def_stack) <= 1: - raise AutogenParseError("Too many close braces", self.cur_line) + self.error("Too many close braces") new_name, parent_def = self.def_stack.pop() parent_def.append((new_name, self.curdef)) self.curdef = parent_def
@@ -292,7 +293,7 @@ class AutogenParser: def do_indexed_name(self, token): self.new_name = token - def read_definitions(self, f): + def read_definitions_file(self, f): self.curdef = self.definitions self.cur_line = 0 state = AutogenState.init
@@ -367,12 +368,17 @@ class AutogenParser: if handler is not None: handler(token) else: - raise AutogenParseError( + self.error( "Parse error in state %s: unexpected token '%s'" % ( - state, token), self.cur_line) + state, token)) if state == AutogenState.done: break + def read_definitions(self, path): + self.cur_file = path + with open(path) as f: + self.read_definitions_file(f) + defparser = AutogenParser() #
@@ -791,8 +797,7 @@ parser = OptionParser(usage="%prog DEFINITION-FILES") _, args = parser.parse_args() for arg in args: - with open(arg) as f: - defparser.read_definitions(f) + defparser.read_definitions(arg) rules("module", module) rules("kernel", kernel)
Signature · 1 line
1.8.4.3
PATCH 3/3 Handle #if/#endif and C-style comments in AutoGen definitions files.
In reply to Colin Watson--- gentpl.py | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-)
gentpl.py +42 -4
@@ -106,9 +106,11 @@ for platform in GRUB_PLATFORMS: # # We support a subset of the AutoGen definitions file syntax. Specifically, -# compound names are disallowed; C-style comments and preprocessing -# directives are disallowed; and shell-generated strings, Scheme-generated -# strings, and here strings are disallowed. +# compound names are disallowed; some preprocessing directives are +# disallowed (though #if/#endif are allowed; note that, like AutoGen, #if +# skips everything to the next #endif regardless of the value of the +# conditional); and shell-generated strings, Scheme-generated strings, and +# here strings are disallowed. class AutogenToken: (autogen, definitions, eof, var_name, other_name, string, number,
@@ -189,7 +191,27 @@ class AutogenParser: if offset >= end: break c = data[offset] - if c == "{": + if c == "#": + offset += 1 + try: + end_directive = data.index("\n", offset) + directive = data[offset:end_directive] + offset = end_directive + except ValueError: + directive = data[offset:] + offset = end + name, value = directive.split(None, 1) + if name == "if": + try: + end_if = data.index("\n#endif", offset) + new_offset = end_if + len("\n#endif") + self.cur_line += data[offset:new_offset].count("\n") + offset = new_offset + except ValueError: + self.error("#if without matching #endif") + else: + self.error("Unhandled directive '#%s'" % name) + elif c == "{": yield AutogenToken.lbrace, c offset += 1 elif c == "=":
@@ -234,6 +256,22 @@ class AutogenParser: else: s.append(data[offset]) yield AutogenToken.string, "".join(s) + elif c == "/": + offset += 1 + if data[offset] == "*": + offset += 1 + try: + end_comment = data.index("*/", offset) + new_offset = end_comment + len("*/") + self.cur_line += data[offset:new_offset].count("\n") + offset = new_offset + except ValueError: + self.error("/* without matching */") + elif data[offset] == "/": + try: + offset = data.index("\n", offset) + except ValueError: + pass elif (c.isdigit() or (c == "-" and offset < end - 1 and data[offset + 1].isdigit())):
Signature · 1 line
1.8.4.3