Pull request for branch for-master/doc

new file mode 100755
index 0000000..760a122
@@ -0,0 +1,354 @@
+#!/usr/bin/env python2
+##
+## gen-manual-lists.py
+##
+## This script generates the following Buildroot manual appendices:
+##  - the package tables (one for the target, the other for host tools);
+##  - the deprecated items.
+##
+## Author(s):
+##  - Samuel Martin <[email protected]>
+##
+## Copyright (C) 2013 Samuel Martin
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##
+
+## Note about python2.
+##
+## This script can currently only be run using python2 interpreter due to
+## its kconfiglib dependency (which is not yet python3 friendly).
+
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import datetime
+from argparse import ArgumentParser
+
+try:
+    import kconfiglib
+except ImportError:
+    message = """
+Could not find the module 'kconfiglib' in the PYTHONPATH:
+"""
+    message += "\n".join(["  {0}".format(path) for path in sys.path])
+    message += """
+
+Make sure the Kconfiglib directory is in the PYTHONPATH, then relaunch the script.
+
+You can get kconfiglib from:
+  https://github.com/ulfalizer/Kconfiglib
+
+
+"""
+    sys.stderr.write(message)
+    raise
+
+
+def get_symbol_subset(root, filter_func):
+    """ Return a generator of kconfig items.
+
+    :param root_item:   Root item of the generated subset of items
+    :param filter_func: Filter function
+
+    """
+    if hasattr(root, "get_items"):
+        get_items = root.get_items
+    elif hasattr(root, "get_top_level_items"):
+        get_items = root.get_top_level_items
+    else:
+        message = "The symbol does not contain any subset of symbols"
+        raise Exception(message)
+    for item in get_items():
+        if item.is_symbol():
+            if not item.prompts:
+                continue
+            if not filter_func(item):
+                continue
+            yield item
+        elif item.is_menu() or item.is_choice():
+            for i in get_symbol_subset(item, filter_func):
+                yield i
+
+
+def get_symbol_parents(item, root=None, enable_choice=False):
+    """ Return the list of the item's parents. The lasst item of the list is
+    the closest parent, the first the furthest.
+
+    :param item:          Item from which the the parent list is generated
+    :param root:          Root item stopping the search (not included in the
+                          parent list)
+    :param enable_choice: Flag enabling choices to appear in the parent list
+
+    """
+    parent = item.get_parent()
+    parents = []
+    while parent and parent != root:
+        if parent.is_menu():
+            parents.append(parent.get_title())
+        elif enable_choice and parent.is_choice():
+            parents.append(parent.prompts[0][0])
+        parent = parent.get_parent()
+    if isinstance(root, kconfiglib.Menu) or \
+            (enable_choice and isinstance(root, kconfiglib.Choice)):
+        parents.append(".")
+    parents.reverse()
+    return parents
+
+
+def format_asciidoc_table(root, get_label_func, filter_func=lambda x: True,
+                          enable_choice=False, sorted=True):
+    """ Return the asciidoc formatted table of the items and their location.
+
+    :param root:           Root item of the item subset
+    :param get_label_func: Item's label getter function
+    :param filter_func:    Filter function to apply on the item subset
+    :param enable_choice:  Enable choices to appear as part of the item's
+                           location
+    :param sorted:         Flag to alphabetically sort the table
+
+    """
+    def _format_entry(label, parents):
+        """ Format an asciidoc table entry.
+
+        """
+        return "| {0:<40} | {1}\n".format(label, " -> ".join(parents))
+
+    lines = []
+    for item in get_symbol_subset(root, filter_func):
+        if not item.is_symbol() or not item.prompts:
+            continue
+        loc = get_symbol_parents(item, root, enable_choice=enable_choice)
+        lines.append(_format_entry(get_label_func(item), loc))
+    if sorted:
+        lines.sort(key=lambda x: x.lower())
+    if hasattr(root, "get_title"):
+        loc_label = get_symbol_parents(root, None, enable_choice=enable_choice)
+        loc_label += [root.get_title(), "..."]
+    else:
+        loc_label = ["Location"]
+    table = "[width=\"90%\",cols=\"^1,4\",options=\"header\"]\n"
+    table += "|===================================================\n"
+    table += _format_entry("Packages", loc_label)
+    table += "\n" + "".join(lines) + "\n"
+    table += "|===================================================\n"
+    return table
+
+
+class Buildroot:
+    """ Buildroot configuration object.
+
+    """
+    root_config = "Config.in"
+    package_dirname = "package"
+    package_prefixes = ["BR2_PACKAGE_", "BR2_PACKAGE_HOST_"]
+    re_pkg_prefix = re.compile(r"^(" + "|".join(package_prefixes) + ").*")
+    deprecated_symbol = "BR2_DEPRECATED"
+
+    list_template = """\
+//
+// Automatically generated list for Buildroot manual.
+// Buildroot {br_version_full}
+// Generation date: {gen_date}
+//
+
+{table}
+"""
+
+    list_info = {
+        'target-packages': {
+            'filename': "package-list",
+            'root_menu': "Package Selection for the target",
+            'filter': "_is_package",
+            'sorted': True,
+        },
+        'host-packages': {
+            'filename': "host-package-list",
+            'root_menu': "Host utilities",
+            'filter': "_is_package",
+            'sorted': True,
+        },
+        'deprecated': {
+            'filename': "deprecated-list",
+            'root_menu': None,
+            'filter': "_is_deprecated",
+            'sorted': False,
+        },
+    }
+
+    use_generator = False
+
+    def __init__(self):
+        self.base_dir = os.environ.get("TOPDIR")
+        self.package_dir = os.path.join(self.base_dir, self.package_dirname)
+        # The kconfiglib requires an environment variable named "srctree" to
+        # load the configuration, so set it.
+        os.environ.update({'srctree': self.base_dir})
+        self.config = kconfiglib.Config(os.path.join(self.base_dir,
+                                                     self.root_config))
+        self._deprecated = self.config.get_symbol(self.deprecated_symbol)
+        self.br_version_full = os.environ.get("BR2_VERSION_FULL")
+        self.generation_date = str(datetime.datetime.utcnow())
+
+    def _get_package_symbols(self, package_name):
+        """ Return a tuple containing the target and host package symbol.
+
+        """
+        symbols = re.sub("[-+.]", "_", package_name)
+        symbols = symbols.upper()
+        symbols = tuple([prefix + symbols for prefix in self.package_prefixes])
+        return symbols
+
+    def _is_deprecated(self, symbol):
+        """ Return True if the symbol is marked as deprecated, otherwise False.
+
+        """
+        return self._deprecated in symbol.get_referenced_symbols()
+
+    def _is_package(self, symbol):
+        """ Return True if the symbol is a package or a host package, otherwise
+        False.
+
+        """
+        if not self.re_pkg_prefix.match(symbol.get_name()):
+            return False
+        pkg_name = re.sub("BR2_PACKAGE_(HOST_)?(.*)", r"\2", symbol.get_name())
+
+        pattern = "^(HOST_)?" + pkg_name + "$"
+        pattern = re.sub("_", ".", pattern)
+        pattern = re.compile(pattern, re.IGNORECASE)
+        # Here, we cannot just check for the location of the Config.in because
+        # of the "virtual" package.
+        #
+        # So, to check that a symbol is a package (not a package option or
+        # anything else), we check for the existence of the package *.mk file.
+        #
+        # By the way, to actually check for a package, we should grep all *.mk
+        # files for the following regex:
+        # "\$\(eval \$\((host-)?(generic|autotools|cmake)-package\)\)"
+        #
+        # Implementation details:
+        #
+        # * The package list is generated from the *.mk file existence, the
+        #   first time this function is called. Despite the memory consumtion,
+        #   this list is stored because the execution time of this script is
+        #   noticebly shorter than re-scannig the package sub-tree for each
+        #   symbol.
+        if not hasattr(self, "_package_list"):
+            pkg_list = []
+            for _, _, files in os.walk(self.package_dir):
+                for file_ in (f for f in files if f.endswith(".mk")):
+                    pkg_list.append(re.sub(r"(.*?)\.mk", r"\1", file_))
+            setattr(self, "_package_list", pkg_list)
+        for pkg in getattr(self, "_package_list"):
+            if pattern.match(pkg):
+                return True
+        return False
+
+    def _get_symbol_label(self, symbol, mark_deprecated=True):
+        """ Return the label (a.k.a. prompt text) of the symbol.
+
+        :param symbol:          The symbol
+        :param mark_deprecated: Append a 'deprecated' to the label
+
+        """
+        label = symbol.prompts[0][0]
+        if self._is_deprecated(symbol) and mark_deprecated:
+            label += " *(deprecated)*"
+        return label
+
+    def print_list(self, list_type, enable_choice=True, enable_deprecated=True,
+                   dry_run=False, output=None):
+        """ Print the requested list. If not dry run, then the list is
+        automatically written in its own file.
+
+        :param list_type:         The list type to be generated
+        :param enable_choice:     Flag enabling choices to appear in the list
+        :param enable_deprecated: Flag enabling deprecated items to appear in
+                                  the package lists
+        :param dry_run:           Dry run (print the list in stdout instead of
+                                  writing the list file
+
+        """
+        def _get_menu(title):
+            """ Return the first symbol menu matching the given title.
+
+            """
+            menus = self.config.get_menus()
+            menu = [m for m in menus if m.get_title().lower() == title.lower()]
+            if not menu:
+                message = "No such menu: '{0}'".format(title)
+                raise Exception(message)
+            return menu[0]
+
+        list_config = self.list_info[list_type]
+        root_title = list_config.get('root_menu')
+        if root_title:
+            root_item = _get_menu(root_title)
+        else:
+            root_item = self.config
+        filter_ = getattr(self, list_config.get('filter'))
+        filter_func = lambda x: filter_(x)
+        if not enable_deprecated and list_type != "deprecated":
+            filter_func = lambda x: filter_(x) and not self._is_deprecated(x)
+        mark_depr = list_type != "deprecated"
+        get_label = lambda x: \
+                        self._get_symbol_label(x, mark_deprecated=mark_depr)
+
+        table = format_asciidoc_table(root_item, get_label,
+                                      filter_func=filter_func,
+                                      enable_choice=enable_choice,
+                                      sorted=list_config.get('sorted'))
+
+        content = self.list_template.format(
+            gen_date=self.generation_date,
+            br_version_full=self.br_version_full,
+            table=table)
+
+        if dry_run:
+            print(content)
+            return
+        if not output:
+            output = os.path.join(self.base_dir, "docs", "manual",
+                                  list_config.get('filename') + ".txt")
+        print("Writing the {0} list in:\n\t{1}".format(list_type, output))
+        with open(output, 'w') as fout:
+            fout.write(content)
+
+
+if __name__ == '__main__':
+    list_types = ['target-packages', 'host-packages', 'deprecated']
+    parser = ArgumentParser()
+    parser.add_argument("list_type", nargs="?", choices=list_types,
+                        help="""\
+Generate the given list (generate all lists if unspecified)""")
+    parser.add_argument("-n", "--dry-run", dest="dry_run", action='store_true',
+                        help="Output the generated list to stdout")
+    parser.add_argument("--output-target", dest="output_target",
+                        help="Output target package file")
+    parser.add_argument("--output-host", dest="output_host",
+                        help="Output host package file")
+    parser.add_argument("--output-deprecated", dest="output_deprecated",
+                        help="Output deprecated file")
+    args = parser.parse_args()
+    lists = [args.list_type] if args.list_type else list_types
+    buildroot = Buildroot()
+    for list_name in lists:
+        output = getattr(args, "output_" + list_name.split("-", 1)[0])
+        buildroot.print_list(list_name, dry_run=args.dry_run,
+                             output=output)

index aa20534..012d0e7 100644
@@ -24,6 +24,11 @@ $$(O)/docs/$(1)/$(1).$(4): docs/$(1)/$(1).txt $$($(call UPPERCASE,$(1))_SOURCES)
 	  -D $$(@D) $$<
 endef
 
+manual-update-lists:
+	@$(call MESSAGE,"Updating the manual lists...")
+	$(Q)BR2_DEFCONFIG="" TOPDIR=$(TOPDIR) \
+		$(TOPDIR)/support/scripts/gen-manual-lists.py
+
 ################################################################################
 # GENDOC -- generates the make targets needed to build asciidoc documentation.
 #

@@ -41,8 +46,9 @@ $(call GENDOC_INNER,$(1),epub,epub,epub,EPUB)
 clean: $(1)-clean
 $(1)-clean:
 	$(Q)$(RM) -rf $(O)/docs/$(1)
-.PHONY: $(1) $(1)-clean
+.PHONY: $(1) $(1)-clean manual-update-lists
 endef
 
 MANUAL_SOURCES = $(wildcard docs/manual/*.txt) $(wildcard docs/images/*)
 $(eval $(call GENDOC,manual))
+

index ef34169..ce54b99 100644
@@ -6,15 +6,26 @@ Appendix
 
 include::makedev-syntax.txt[]
 
+
+// Automatically generated lists:
+
 [[package-list]]
-Available packages
-------------------
-// docs/manaual/pkg-list.txt is generated using the following command:
-// $ git grep -E '\((autotools|cmake|generic)-package\)' package/ |  \
-//     cut -d':' -f1 | grep '\.mk$' | \
-//     sed -e 's;.*\?/\(.*\?\).mk;* \1;' | \
-//     sort > docs/manual/pkg-list.txt
-
-include::pkg-list.txt[]
+Package Selection for the target
+--------------------------------
+
+include::package-list.txt[]
+
+[[host-package-list]]
+Host utilities
+--------------
+
+include::host-package-list.txt[]
+
+[[deprecated-list]]
+Deprecated stuff
+----------------
+
+The following stuff are marked as _deprecated_ in Buildroot due to
+their status either too old or unmaintained.
 
 include::deprecated-list.txt[]

index c462bb1..2b25059 100644
@@ -817,6 +817,8 @@ release: OUT=buildroot-$(BR2_VERSION)
 # Create release tarballs. We need to fiddle a bit to add the generated
 # documentation to the git output
 release:
+	$(warning Make sure the manual is up-to-date. To update the manual, run:)
+	$(warning $$ make manual-update-lists)
 	git archive --format=tar --prefix=$(OUT)/ master > $(OUT).tar
 	$(MAKE) O=$(OUT) manual-html manual-txt manual-pdf
 	tar rf $(OUT).tar $(OUT)

index 6dc87a4..d8489b5 100644
@@ -1,46 +1,39 @@
-// -*- mode:doc -*- ;
-
-[[deprecated]]
-Deprecated list
----------------
-
-The following stuff are marked as _deprecated_ in Buildroot due to
-their status either too old or unmaintained.
-
-// list generated using the followings command:
-// $ git grep -EB4 'depends on BR2_DEPRECATED'
-// and
-// $ git grep -EB4 'depends on BR2_DEPRECATED' | \
-//     grep -Eo '(:|-).*?(config|comment) BR2_.*'
 //
-// Need manual checks and sorting.
-
-* Packages:
-
-** +busybox+ 1.18.x
-** +customize+
-** +lzma+
-** +microperl+
-** +netkitbase+
-** +netkittelnet+
-** +pkg-config+
-** +squashfs3+
-** +ttcp+
-
-* Toolchain:
-
-** +gdb+ 6.8
-** +gdb+ 7.0.1
-** +gdb+ 7.1
-** +kernel headers+ 2.6.37
-** +kernel headers+ 2.6.38
-** +kernel headers+ 2.6.39
-
-* Bootloaders:
-
-** +u-boot+ 2011-06
-** +u-boot+ 2011-09
+// Automatically generated list for Buildroot manual.
+// Buildroot 2013.05-git-00117-gfecb9e5
+// Generation date: 2013-03-12 04:40:24.263952
+//
 
-* Output images:
+[width="90%",cols="^1,4",options="header"]
+|===================================================
+| Packages                                 | Location
+
+| documentation on the target              | Build options
+| development files in target filesystem   | Build options
+| Linux 3.1.x kernel headers               | Toolchain -> Kernel Headers
+| Linux 3.3.x kernel headers               | Toolchain -> Kernel Headers
+| Linux 3.5.x kernel headers               | Toolchain -> Kernel Headers
+| Linux 3.6.x kernel headers               | Toolchain -> Kernel Headers
+| uClibc 0.9.31.x                          | Toolchain -> uClibc C library Version
+| binutils 2.20                            | Toolchain -> Binutils Version
+| Build gdb debugger for the Target        | Toolchain
+| gdb 7.2.x                                | Toolchain -> GDB debugger Version
+| gdb 7.3.x                                | Toolchain -> GDB debugger Version
+| lzma                                     | Package Selection for the target -> Compressors and decompressors
+| autoconf                                 | Package Selection for the target -> Development tools
+| automake                                 | Package Selection for the target -> Development tools
+| ccache                                   | Package Selection for the target -> Development tools
+| gcc                                      | Package Selection for the target -> Development tools
+| make                                     | Package Selection for the target -> Development tools
+| pkg-config                               | Package Selection for the target -> Development tools
+| xstroke                                  | Package Selection for the target -> Graphic libraries and applications (graphic/text)
+| squashfs3                                | Package Selection for the target -> Filesystem and flash utilities
+| netkitbase                               | Package Selection for the target -> Networking applications
+| netkittelnet                             | Package Selection for the target -> Networking applications
+| ttcp                                     | Package Selection for the target -> Networking applications
+| 3.x                                      | Filesystem images -> SquashFS version
+| 2011.09                                  | Bootloaders -> U-Boot Version
+| 2011.06                                  | Bootloaders -> U-Boot Version
+
+|===================================================
 
-** squashfs3 image

new file mode 100644
index 0000000..c8a01a5
@@ -0,0 +1,19 @@
+//
+// Automatically generated list for Buildroot manual.
+// Buildroot 2013.05-git-00117-gfecb9e5
+// Generation date: 2013-03-12 04:40:24.263952
+//
+
+[width="90%",cols="^1,4",options="header"]
+|===================================================
+| Packages                                 | Host utilities -> ...
+
+| host dfu-util                            | .
+| host lpc3250loader                       | .
+| host omap-u-boot-utils                   | .
+| host openocd                             | .
+| host sam-ba                              | .
+| host u-boot tools                        | .
+
+|===================================================
+

941 lines not on this page · raw message

870 lines not on this page · raw message