br2-external: support multiple trees at once (branch yem/multi-br2-external-10)

Now that we generate a kconfig snippet, we can conditionally include the BR2_EXTERNAL's Config.in only when BR2_EXTERNAL is supplied by the user, which means our empty/dummy Config.in is no longer useful to us.

As for external.mk, we can also include it only when BR2_EXTERNAL is supplied by the user, which means our empty/dummy external.mk is no longer of any use to us.

Ditch both of those files, and:

- only generate actual content in the Kconfig snippet when we actually do have a BR2_EXTERNAL provided by the user (i.e. BR2_EXTERNAL is not empty);

- add a variable that contains the path to the external.mk provided by the user, or empty if none, and include the path set in that variable (make can 'include' nothing without any problem! ;-) )

Signed-off-by "Yann E. MORIN" <[email protected]>

Cc Thomas Petazzoni <[email protected]>

Cc Arnout Vandecappelle <[email protected]>

Cc Romain Naour <[email protected]>

---
 Makefile                           | 11 ++++-------
 support/dummy-external/Config.in   |  0
 support/dummy-external/external.mk |  0
 support/scripts/br2-external       | 15 +++++++++++----
 4 files changed, 15 insertions(+), 11 deletions(-)
 delete mode 100644 support/dummy-external/Config.in
 delete mode 100644 support/dummy-external/external.mk

Makefile +4 -7

index 513a989..8d5ce8a 100644
@@ -147,16 +147,11 @@ $(if $(BASE_DIR),, $(error output directory "$(O)" does not exist))
 # on the command line, therefore the file is re-created every time make is run.
 #
 # When BR2_EXTERNAL is set to an empty value (e.g. explicitly in command
-# line), the .br-external file is removed and we point to
-# support/dummy-external. This makes sure we can unconditionally include the
-# Config.in and external.mk from the BR2_EXTERNAL directory. In this case,
-# override is necessary so the user can clear BR2_EXTERNAL from the command
-# line, but the dummy path is still used internally.
+# line), the .br-external file is removed.
 
 BR2_EXTERNAL_FILE = $(BASE_DIR)/.br-external
 -include $(BR2_EXTERNAL_FILE)
 ifeq ($(BR2_EXTERNAL),)
-  override BR2_EXTERNAL = support/dummy-external
   $(shell rm -f $(BR2_EXTERNAL_FILE))
 else
   _BR2_EXTERNAL = $(shell cd $(BR2_EXTERNAL) >/dev/null 2>&1 && pwd)

@@ -165,6 +160,7 @@ else
   endif
   override BR2_EXTERNAL := $(_BR2_EXTERNAL)
   $(shell echo BR2_EXTERNAL ?= $(BR2_EXTERNAL) > $(BR2_EXTERNAL_FILE))
+  BR2_EXTERNAL_MK = $(BR2_EXTERNAL)/external.mk
 endif
 
 # To make sure that the environment variable overrides the .config option,

@@ -461,7 +457,8 @@ include boot/common.mk
 include linux/linux.mk
 include fs/common.mk
 
-include $(BR2_EXTERNAL)/external.mk
+# Nothing to include if no BR2_EXTERNAL tree in use
+include $(BR2_EXTERNAL_MK)
 
 # Now we are sure we have all the packages scanned and defined. We now
 # check for each package in the list of enabled packages, that all its

support/dummy-external/Config.in deleted +0 -0

deleted file mode 100644
index e69de29..0000000

support/dummy-external/external.mk deleted +0 -0

deleted file mode 100644
index e69de29..0000000

support/scripts/br2-external +0 -0

index c15c21c..91f854a 100755

support/scripts/br2-external +11 -1

     br2_ext="${1}"
 
     if [ -z "${ofile}" ]; then

@@ -38,6 +35,11 @@ main() {
 do_validate() {
     local br2_ext="${1}"
 
+    # No br2-external tree is valid
+    if [ -z "${br2_ext}" ]; then
+        return
+    fi
+
     if [ ! -d "${br2_ext}" ]; then
         error "'%s': no such file or directory\n" "${br2_ext}"
     fi

@@ -49,12 +51,17 @@ do_validate() {
 do_kconfig() {
     printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
     printf '\n'
+
+    if [ -z "${BR2_EXT}" ]; then
+        printf '# No br2-external tree defined.\n'
+        return
+    fi
+
     printf 'config BR2_EXTERNAL\n'
     printf '\tstring\n'
     printf '\tdefault "%s"\n' "${BR2_EXT}"
     printf '\n'
     printf 'menu "User-provided options"\n'
-    printf '\tdepends on BR2_EXTERNAL != "support/dummy-external"\n'
     printf '\n'
     printf 'source "%s/Config.in"\n' "${BR2_EXT}"
     printf '\n'

I've just done a static read of your series and my comments are just superficial as I'm not a kconfig/Makefile expert...

I will test all theses before any review/test tag but I do not have enough time yet to do this in the few next days.

Quoted 8 lines

support/scripts/br2-external +0 -0

index c15c21c..91f854a 100755
@@ -19,9 +19,6 @@ main() {
     # Forget options; keep only positional args
     shift $((OPTIND-1))
 

support/scripts/br2-external main() { · line 24

Quoted 3 lines

support/scripts/br2-external +0 -3

-    if [ ${#} -ne 1 ]; then
-        error "need exactly one br2-external tree to be specified\n"
-    fi

I don't understand why do you remove this check in this commit rather than in patch "7 - core: add support for multiple br2-external trees"?

Regards,

Julien Corjon

Quoted 25 lines

support/scripts/br2-external +10 -0

     br2_ext="${1}"
 
     if [ -z "${ofile}" ]; then

@@ -38,6 +35,11 @@ main() {
 do_validate() {
     local br2_ext="${1}"
 
+    # No br2-external tree is valid
+    if [ -z "${br2_ext}" ]; then
+        return
+    fi
+
     if [ ! -d "${br2_ext}" ]; then
         error "'%s': no such file or directory\n" "${br2_ext}"
     fi

@@ -49,12 +51,17 @@ do_validate() {
 do_kconfig() {
     printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
     printf '\n'
+
+    if [ -z "${BR2_EXT}" ]; then
+        printf '# No br2-external tree defined.\n'
+        return
+    fi

support/scripts/br2-external do_validate() { · line 67

Quoted 10 lines

support/scripts/br2-external +1 -1

+
     printf 'config BR2_EXTERNAL\n'
     printf '\tstring\n'
     printf '\tdefault "%s"\n' "${BR2_EXT}"
     printf '\n'
     printf 'menu "User-provided options"\n'
-    printf '\tdepends on BR2_EXTERNAL != "support/dummy-external"\n'
     printf '\n'
     printf 'source "%s/Config.in"\n' "${BR2_EXT}"
     printf '\n'

_______________________________________________
buildroot mailing list
buildroot at busybox.net
http://lists.busybox.net/mailman/listinfo/buildroot

Update the manual with the new external.desc mandatory file.

Take the opportunity to add a section listing all mandatory files, Config.in, external.mk and the new external.desc, instead of just hinting about them in the external package recipes section.

Change the examples to use the NAME-suffixed variable instead of the raw BR2_EXTERNAL variable. Even though it is still possible to write a br2-external tree that has no NAME (ie.e. won't be multi-aware), we only document it with an NAME.

Change all references to BR2_EXTERNAL elsewhere in the manual to now use the 'br2-external tree' terminology.

Signed-off-by "Yann E. MORIN" <[email protected]>

Cc Thomas Petazzoni <[email protected]>

Cc Arnout Vandecappelle <[email protected]>

Cc Samuel Martin <[email protected]>

Cc Romain Naour <[email protected]>

---
 docs/manual/adding-packages-asciidoc.txt      |   6 +-
 docs/manual/adding-packages-perl.txt          |   2 +-
 docs/manual/adding-packages-python.txt        |   2 +-
 docs/manual/customize-directory-structure.txt |  13 +--
 docs/manual/customize-outside-br.txt          | 134 ++++++++++++++++----------
 docs/manual/customize-packages.txt            |  25 +----
 6 files changed, 102 insertions(+), 80 deletions(-)

docs/manual/adding-packages-asciidoc.txt +3 -3

index a278d44..d870c51 100644
@@ -19,9 +19,9 @@ Although Buildroot only contains one document written in AsciiDoc, there
 is, as for packages, an infrastructure for rendering documents using the
 AsciiDoc syntax.
 
-Also as for packages, the AsciiDoc infrastructure is available from
-xref:outside-br-custom[BR2_EXTERNAL]. This allows documentation for a
-BR2_EXTERNAL tree to match the Buildroot documentation, as it will be
+Also as for packages, the AsciiDoc infrastructure is available from a
+xref:outside-br-custom[br2-external tree]. This allows documentation for
+a br2-external tree to match the Buildroot documentation, as it will be
 rendered to the same formats and use the same layout and theme.
 
 ==== +asciidoc-document+ tutorial

docs/manual/adding-packages-perl.txt +1 -1

index 63fafe6..4f5a6a4 100644
@@ -47,7 +47,7 @@ built.
 Most of these data can be retrieved from https://metacpan.org/.
 So, this file and the Config.in can be generated by running
 the script +supports/scripts/scancpan Foo-Bar+ in the Buildroot directory
-(or in the +BR2_EXTERNAL+ directory).
+(or in a br2-external tree).
 This script creates a Config.in file and foo-bar.mk file for the
 requested package, and also recursively for all dependencies specified by
 CPAN. You should still manually edit the result. In particular, the

docs/manual/adding-packages-python.txt +1 -1

index 94ac809..abcd4ca 100644
@@ -195,7 +195,7 @@ license and license files are guessed and must be checked. You also
 need to manually add the package to the +package/Config.in+ file.
 
 If your Buildroot package is not in the official Buildroot tree but in
-a +BR2_EXTERNAL+ tree, use the -o flag as follows:
+a br2-external tree, use the -o flag as follows:
 
 -----------------------
 ./support/script/scanpypi foo bar -o other_package_dir

docs/manual/customize-directory-structure.txt +7 -6

index 0be3f77..b177319 100644
@@ -13,7 +13,8 @@ section.
 
 Orthogonal to this directory structure, you can choose _where_ you place
 this structure itself: either inside the Buildroot tree, or outside of
-it using +BR2_EXTERNAL+. Both options are valid, the choice is up to you.
+it using a br2-external tree. Both options are valid, the choice is up
+to you.
 
 -----
 +-- board/

@@ -38,8 +39,8 @@ it using +BR2_EXTERNAL+. Both options are valid, the choice is up to you.
 |
 +-- package/
 |   +-- <company>/
-|       +-- Config.in (if not using BR2_EXTERNAL)
-|       +-- <company>.mk (if not using BR2_EXTERNAL)
+|       +-- Config.in (if not using a br2-external tree)
+|       +-- <company>.mk (if not using a br2-external tree)
 |       +-- package1/
 |       |    +-- Config.in
 |       |    +-- package1.mk

@@ -47,14 +48,14 @@ it using +BR2_EXTERNAL+. Both options are valid, the choice is up to you.
 |           +-- Config.in
 |           +-- package2.mk
 |
-+-- Config.in (if using BR2_EXTERNAL)
-+-- external.mk (if using BR2_EXTERNAL)
++-- Config.in (if using a br2-external tree)
++-- external.mk (if using a br2-external tree)
 ------
 
 Details on the files shown above are given further in this chapter.
 
 Note: if you choose to place this structure outside of the Buildroot
-tree using +BR2_EXTERNAL+, the <company> and possibly <boardname>
+tree but in a br2-external tree, the <company> and possibly <boardname>
 components may be superfluous and can be left out.
 
 ==== Implementing layered customizations

docs/manual/customize-outside-br.txt +85 -49

index 9ad177d..f2a83a6 100644
@@ -11,25 +11,27 @@ place project-specific customizations in two locations:
    branches in a version control system so that upgrading to a newer
    Buildroot release is easy.
 
- * outside of the Buildroot tree, using the +BR2_EXTERNAL+ mechanism.
+ * outside of the Buildroot tree, using the _br2-external_ mechanism.
    This mechanism allows to keep package recipes, board support and
    configuration files outside of the Buildroot tree, while still
-   having them nicely integrated in the build logic. This section
-   explains how to use +BR2_EXTERNAL+.
+   having them nicely integrated in the build logic. We call this
+   location a _br2-external tree_. This section explains how to use
+   the br2-external mechanism and what to provide in a br2-external
+   tree.
 
-+BR2_EXTERNAL+ is an environment variable that can be used to point to
-a directory that contains Buildroot customizations. It can be passed
-to any Buildroot +make+ invocation. It is automatically saved in the
-hidden +.br-external+ file in the output directory. Thanks to this,
-there is no need to pass +BR2_EXTERNAL+ at every +make+ invocation. It
-can however be changed at any time by passing a new value, and can be
-removed by passing an empty value.
+One can tell Buildroot to use a br2-external tree by setting the
++BR2_EXTERNAL+ make variable set to the path of the br2-external tree
+to use. It can be passed to any Buildroot +make+ invocation. It is
+automatically saved in the hidden +.br-external.mk+ file in the output
+directory. Thanks to this, there is no need to pass +BR2_EXTERNAL+ at
+every +make+ invocation. It can however be changed at any time by
+passing a new value, and can be removed by passing an empty value.
 
 .Note
-The +BR2_EXTERNAL+ path can be either an absolute or a relative path,
-but if it's passed as a relative path, it is important to note that it
-is interpreted relative to the main Buildroot source directory, *not*
-to the Buildroot output directory.
+The path to a br2-external tree can be either absolute or relative.
+If it is passed as a relative path, it is important to note that it is
+interpreted relative to the main Buildroot source directory, *not* to
+the Buildroot output directory.
 
 Some examples:
 

@@ -37,73 +39,107 @@ Some examples:
 buildroot/ $ make BR2_EXTERNAL=/path/to/foobar menuconfig
 -----
 
-From now on, external definitions from the +/path/to/foobar+
-directory will be used:
+From now on, definitions from the +/path/to/foobar+ br2-external tree
+will be used:
 
 -----
 buildroot/ $ make
 buildroot/ $ make legal-info
 -----
 
-We can switch to another external definitions directory at any time:
+We can switch to another br2-external tree at any time:
 
 -----
 buildroot/ $ make BR2_EXTERNAL=/where/we/have/barfoo xconfig
 -----
 
-Or disable the usage of external definitions:
+Or disable the usage of any br2-external tree:
 
 -----
 buildroot/ $ make BR2_EXTERNAL= xconfig
 -----
 
-+BR2_EXTERNAL+ allows three different things:
+A br2-external tree must contain at least those three files:
 
- * One can store all the board-specific configuration files there,
-   such as the kernel configuration, the root filesystem overlay, or
-   any other configuration file for which Buildroot allows to set its
-   location. The +BR2_EXTERNAL+ value is available within the
-   Buildroot configuration using +$(BR2_EXTERNAL)+. As an example, one
-   could set the +BR2_ROOTFS_OVERLAY+ Buildroot option to
-   +$(BR2_EXTERNAL)/board/<boardname>/overlay/+ (to specify a root
-   filesystem overlay), or the +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE+
-   Buildroot option to
-   +$(BR2_EXTERNAL)/board/<boardname>/kernel.config+ (to specify the
-   location of the kernel configuration file).
++external.desc+::
+   That file shall contain the _name_ for the br2-external tree. That name
+   must only use ASCII characters in the set +[A-Za-z0-9_]+; any other
+   character is forbidden. The format for this file is a single line with
+   the keyword 'name:', followed by one or more spaces, followed by the
+   name.
++
+Buildroot sets +BR2_EXTERNAL_$(NAME)_PATH+ to the absolute path of the
+   br2-external tree, so that you can use it to refer to your br2-external
+   tree. This variable is available both in Kconfig, so you can use it
+   to source your Kconfig files (see below) and in the Makefile, so that
+   you can use it to include other Makefiles (see below) or refer to other
+   files (like data files) from your br2-external tree.
++
+Example of an +external.desc+ file that declares the name +FOO+:
++
+----
+$ cat external.desc
+name: FOO
+----
++
+Examples of names and the corresponding +BR2_EXTERNAL_$(NAME)_PATH+
+variables:
++
+  * +FOO+ -> +BR2_EXTERNAL_FOO_PATH+
+  * +BAR_42+ -> +BR2_EXTERNAL_BAR_42_PATH+
++
+In the following examples, it is assumed the name to be set to +BAR_42+.
+
++Config.in+::
++external.mk+::
+   Those files (which may each be empty) can be used to define package
+   recipes, like for packages bundled in Buildroot itself, or other
+   custom configuration options.
 
- * One can store package recipes (i.e. +Config.in+ and
-   +<packagename>.mk+), or even custom configuration options and make
-   logic. Buildroot automatically includes +$(BR2_EXTERNAL)/Config.in+ to
-   make it appear in the top-level configuration menu, and includes
-   +$(BR2_EXTERNAL)/external.mk+ with the rest of the makefile logic.
+Using a br2-external tree then allows three different things:
+
+ * One can store all the board-specific configuration files there, such
+   as the kernel configuration, the root filesystem overlay, or any other
+   configuration file for which Buildroot allows to set the location (by
+   using the +BR2_EXTERNAL_$(NAME)_PATH+ variable). For example, you
+   could set the paths to a global patch directory, to a rootfs overlay
+   and to the kernel configuration file as follows (e.g. by running
+   `make menuconfig` and filling in these options):
 +
-.Note
-Providing +Config.in+ and +external.mk+ is mandatory, but they can be
-   empty.
+----
+BR2_GLOBAL_PATCH_DIR=$(BR2_EXTERNAL_BAR_42_PATH)/patches/
+BR2_ROOTFS_OVERLAY=$(BR2_EXTERNAL_BAR_42_PATH)/board/<boardname>/overlay/
+BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=$(BR2_EXTERNAL_BAR_42_FOO)/board/<boardname>/kernel.config
+----
+
+ * One can store package recipes (i.e. +Config.in+ and +<packagename>.mk+),
+   or even custom configuration options and make logic. Buildroot
+   automatically includes +Config.in+ to make it appear in the top-level
+   configuration menu, and includes +external.mk+ with the rest of the
+   makefile logic.
 +
-The main usage of this is to store package recipes. The recommended
-   way to do this is to write a +$(BR2_EXTERNAL)/Config.in+ file that
-   looks like:
+The main usage of this is to store package recipes. The recommended way
+   to do this is to write a +Config.in+ file that looks like:
 +
 ------
-source "$BR2_EXTERNAL/package/package1/Config.in"
-source "$BR2_EXTERNAL/package/package2/Config.in"
+source "$BR2_EXTERNAL_BAR_42_PATH/package/package1/Config.in"
+source "$BR2_EXTERNAL_BAR_42_PATH/package/package2/Config.in"
 ------
 +
-Then, have a +$(BR2_EXTERNAL)/external.mk+ file that looks like:
+Then, have an +external.mk+ file that looks like:
 +
 ------
-include $(sort $(wildcard $(BR2_EXTERNAL)/package/*/*.mk))
+include $(sort $(wildcard $(BR2_EXTERNAL_BAR_42_PATH)/package/*/*.mk))
 ------
 +
-And then in +$(BR2_EXTERNAL)/package/package1+ and
-   +$(BR2_EXTERNAL)/package/package2+ create normal Buildroot
-   package recipes, as explained in xref:adding-packages[].
+And then in +$(BR2_EXTERNAL_FOO_42_PATH)/package/package1+ and
+   +$(BR2_EXTERNAL_FOO_42_PATH)/package/package2+ create normal
+   Buildroot package recipes, as explained in xref:adding-packages[].
    If you prefer, you can also group the packages in subdirectories
    called <boardname> and adapt the above paths accordingly.
 
  * One can store Buildroot defconfigs in the +configs+ subdirectory of
-   +$(BR2_EXTERNAL)+. Buildroot will automatically show them in the
+   the br2-external tree. Buildroot will automatically show them in the
    output of +make list-defconfigs+ and allow them to be loaded with the
    normal +make <name>_defconfig+ command. They will be visible under the
    +User-provided configs+' label in the 'make list-defconfigs' output.

docs/manual/customize-packages.txt +5 -20

index 9a5e8c5..b57280e 100644
@@ -14,8 +14,9 @@ packages in a project-specific directory.
 
 As shown in xref:customize-dir-structure[], the recommended location for
 project-specific packages is +package/<company>/+. If you are using the
-+BR2_EXTERNAL+ feature (see xref:outside-br-custom[]) the recommended
-location is +$(BR2_EXTERNAL)/package/+.
+br2-external tree feature (see xref:outside-br-custom[]) the recommended
+location is to put them in a sub-directory named +package/+ in your
+br2-external tree.
 
 However, Buildroot will not be aware of the packages in this location,
 unless we perform some additional steps. As explained in

@@ -37,14 +38,6 @@ have only one extra directory level below +package/<company>/+):
 include $(sort $(wildcard package/<company>/*/*.mk))
 -----
 
-If you are using +BR2_EXTERNAL+, create a file
-+$(BR2_EXTERNAL)/external.mk+ with following contents (again assuming only
-one extra level):
-
------
-include $(sort $(wildcard $(BR2_EXTERNAL)/package/*/*.mk))
------
-
 For the +Config.in+ files, create a file +package/<company>/Config.in+
 that includes the +Config.in+ files of all your packages. An exhaustive
 list has to be provided since wildcards are not supported in the source command of kconfig.

@@ -59,13 +52,5 @@ Include this new file +package/<company>/Config.in+ from
 +package/Config.in+, preferably in a company-specific menu to make
 merges with future Buildroot versions easier.
 
-If you are using +BR2_EXTERNAL+, create a file
-+$(BR2_EXTERNAL)/Config.in+ with similar contents:
-
------
-source "$BR2_EXTERNAL/package/package1/Config.in"
-source "$BR2_EXTERNAL/package/package2/Config.in"
------
-
-You do not have to add an include for this +$(BR2_EXTERNAL)/Config.in+
-file as it is included automatically.
+If using a br2-external tree, refer to xref:outside-br-custom[] for how
+to fill in those files.

Le 05/09/2016 ? 23:49, Yann E. MORIN a ?crit :

Quoted 6 lines

Update the manual with the new external.desc mandatory file.

Take the opportunity to add a section listing all mandatory files, Config.in, external.mk and the new external.desc, instead of just hinting about them in the external package recipes section.

Quoted 4 lines

Change the examples to use the NAME-suffixed variable instead of the raw BR2_EXTERNAL variable. Even though it is still possible to write a br2-external tree that has no NAME (ie.e. won't be multi-aware), we only document it with an NAME.

Is it really possible to have an br2-external tree that has no NAME from here? Or, as you said in the series resume it's not possible anymore?

Currently, we only support at most one br2-external tree. Being able to use more than one br2-external tree can be very useful.

A use-case would be for having a br2-external to contain the basic packages, basic board defconfigs and board files, provided by one team responsible for the "board-bringup", while other teams consume that br2-external as a base, and complements it each with their own set of packages, defconfigs and extra board files.

Another use-case would be for third-parties to provide their own Buildroot packaging in a br2-external tree, along-side the archives for their stuff.

Finally, another use-case is to be able to add FLOSS packages in a br2-external tree, and proprietary packages in another. This allows to not touch the Buildroot tree at all, and still be able to get in compliance by providing only that br2-external tree(s) that contains FLOSS packages, leaving aside the br2-external tree(s) with the proprietary bits.

What we do is to treat BR2_EXTERNAL as a colon-separated (space- separated also work, and we use that internally) list of paths, on which we iterate to construct:

- the list of all br2-external names, BR2_EXTERNAL_NAMES,

- the per-br2-external tree BR2_EXTERNAL_$(NAME) variables, which point each to the actual location of the corresponding tree,

- the list of paths to all the external.mk files, BR2_EXTERNAL_MKS,

- the space-separated list of absolute paths to the external trees, BR2_EXTERNAL_DIRS.

Once we have all those variables, we replace references to BR2_EXTERNAL with either one of those.

This cascades into how we display the list of defconfigs, so that it is easy to see what br2-external tree provides what defconfigs. As suggested by Arnout, tweak the comment from "User-provided configs" to "External configs", on the assumption that some br2-external trees could be provided by vendors, so not necessarily user-provided. Ditto the menu in Kconfig, changed from "User-provided options" to "External options".

Now, when more than one br2-external tree is used, each gets its own sub-menu in the "User-provided options" menu. The sub-menu is labelled with that br2-external tree's name and the sub-menu's first item is a comment with the path to that br2-external tree.

If there's only one br2-external tree, then there is no sub-menu; there is a single comment that contains the name and path to the br2-external tree.

Signed-off-by "Yann E. MORIN" <[email protected]>

Cc Thomas Petazzoni <[email protected]>

Cc Arnout Vandecappelle <[email protected]>

Cc Romain Naour <[email protected]>

Cc Julien CORJON <[email protected]>

---
 Makefile                     |  57 +++++++++++++---------
 package/pkg-generic.mk       |   4 +-
 support/scripts/br2-external | 112 +++++++++++++++++++++++++++++--------------
 3 files changed, 113 insertions(+), 60 deletions(-)

Makefile +35 -22

index 74ed266..c0edb00 100644
@@ -143,7 +143,7 @@ $(if $(BASE_DIR),, $(error output directory "$(O)" does not exist))
 # Handling of BR2_EXTERNAL.
 #
 # The value of BR2_EXTERNAL is stored in .br-external in the output directory.
-# The location of the external.mk makefile fragment is computed in that file.
+# The location of the external.mk makefile fragments is computed in that file.
 # On subsequent invocations of make, this file is read in. BR2_EXTERNAL can
 # still be overridden on the command line, therefore the file is re-created
 # every time make is run.

@@ -454,16 +454,15 @@ include boot/common.mk
 include linux/linux.mk
 include fs/common.mk
 
-# If using a br2-external tree, the BR2_EXTERNAL_$(NAME)_PATH variable
-# is also present in the .config file. Since .config is included after
-# we defined BR2_EXTERNAL_$(NAME)_PATH in the Makefile, the value in
-# that variable is quoted. We just include the generated Makefile fragment
-# .br2-external.mk a third time, which will set that variable to the
-# un-quoted value.
+# If using a br2-external tree, the BR2_EXTERNAL_$(NAME)_PATH variables
+# are also present in the .config file. Since .config is included after
+# we defined them in the Makefile, the values for those variables are
+# quoted. We just include the generated Makefile fragment .br2-external.mk
+# a third time, which will set those variables to the un-quoted values.
 include $(BR2_EXTERNAL_FILE)
 
 # Nothing to include if no BR2_EXTERNAL tree in use
-include $(BR2_EXTERNAL_MK)
+include $(BR2_EXTERNAL_MKS)
 
 # Now we are sure we have all the packages scanned and defined. We now
 # check for each package in the list of enabled packages, that all its

@@ -858,7 +857,7 @@ define percent_defconfig
 	@$$(COMMON_CONFIG_ENV) BR2_DEFCONFIG=$(1)/configs/$$@ \
 		$$< --defconfig=$(1)/configs/$$@ $$(CONFIG_CONFIG_IN)
 endef
-$(eval $(foreach d,$(TOPDIR) $(if $(BR2_EXTERNAL_NAME),$(BR2_EXTERNAL_$(BR2_EXTERNAL_NAME)_PATH)),$(call percent_defconfig,$(d))$(sep)))
+$(eval $(foreach d,$(TOPDIR) $(BR2_EXTERNAL_DIRS),$(call percent_defconfig,$(d))$(sep)))
 
 savedefconfig: $(BUILD_DIR)/buildroot-config/conf prepare-kconfig
 	@$(COMMON_CONFIG_ENV) $< \

@@ -984,19 +983,33 @@ help:
 	@echo 'it on-line at http://buildroot.org/docs.html'
 	@echo
 
+# List the defconfig files
+# $(1): base directory
+# $(2): br2-external name, empty for bundled
+define list-defconfigs
+	@first=y; \
+	for defconfig in $(1)/configs/*_defconfig; do \
+		[ -f "$${defconfig}" ] || continue; \
+		if [ "$${first}" ]; then \
+			if [ "$(2)" ]; then \
+				printf "External configs in $(2):\n"; \
+			else \
+				printf "Built-in configs:\n"; \
+			fi; \
+			first=; \
+		fi; \
+		defconfig="$${defconfig##*/}"; \
+		printf "  %-35s - Build for %s\n" "$${defconfig}" "$${defconfig%_defconfig}"; \
+	done;\
+	printf "\n"
+endef
+
+# We iterate over BR2_EXTERNAL_NAMES rather than BR2_EXTERNAL_DIRS,
+# because we want to display the name of the br2-external tree.
 list-defconfigs:
-	@echo 'Built-in configs:'
-	@$(foreach b, $(sort $(notdir $(wildcard $(TOPDIR)/configs/*_defconfig))), \
-	  printf "  %-35s - Build for %s\\n" $(b) $(b:_defconfig=);)
-ifneq ($(BR2_EXTERNAL_NAME),)
-ifneq ($(wildcard $(BR2_EXTERNAL_$(BR2_EXTERNAL_NAME)_PATH)/configs/*_defconfig),)
-	@echo
-	@echo 'User-provided configs:'
-	@$(foreach b, $(sort $(notdir $(wildcard $(BR2_EXTERNAL_$(BR2_EXTERNAL_NAME)_PATH)/configs/*_defconfig))), \
-	  printf "  %-35s - Build for %s\\n" $(b) $(b:_defconfig=);)
-endif
-endif
-	@echo
+	$(call list-defconfigs,$(TOPDIR))
+	$(foreach name,$(BR2_EXTERNAL_NAMES),\
+		$(call list-defconfigs,$(BR2_EXTERNAL_$(name)_PATH),$(name))$(sep))
 
 release: OUT = buildroot-$(BR2_VERSION)
 

@@ -1015,7 +1028,7 @@ print-version:
 	@echo $(BR2_VERSION_FULL)
 
 include docs/manual/manual.mk
--include $(if $(BR2_EXTERNAL_NAME),$(BR2_EXTERNAL_$(BR2_EXTERNAL_NAME)_PATH)/docs/*/*.mk)
+-include $(foreach dir,$(BR2_EXTERNAL_DIRS),$(dir)/docs/*/*.mk)
 
 .PHONY: $(noconfig_targets)
 

package/pkg-generic.mk +2 -2

index 946244e..0737fa1 100644
@@ -767,9 +767,9 @@ $$($(2)_TARGET_DIRCLEAN):		PKG=$(2)
 # kernel case, the bootloaders case, and the normal packages case.
 ifeq ($(1),linux)
 $(2)_KCONFIG_VAR = BR2_LINUX_KERNEL
-else ifneq ($$(filter boot/% $$(if $$(BR2_EXTERNAL_NAME),$$(BR2_EXTERNAL_$$(BR2_EXTERNAL_NAME)_PATH)/boot/%),$(pkgdir)),)
+else ifneq ($$(filter boot/% $$(foreach dir,$$(BR2_EXTERNAL_DIRS),$$(dir)/boot/%),$(pkgdir)),)
 $(2)_KCONFIG_VAR = BR2_TARGET_$(2)
-else ifneq ($$(filter toolchain/% $$(if $$(BR2_EXTERNAL_NAME),$$(BR2_EXTERNAL_$$(BR2_EXTERNAL_NAME)_PATH)/toolchain/%),$(pkgdir)),)
+else ifneq ($$(filter toolchain/% $$(foreach dir,$$(BR2_EXTERNAL_DIRS),$$(dir)/toolchain/%),$(pkgdir)),)
 $(2)_KCONFIG_VAR = BR2_$(2)
 else
 $(2)_KCONFIG_VAR = BR2_PACKAGE_$(2)

support/scripts/br2-external +3 -5

index 18e547e..9936650 100755
@@ -1,9 +1,9 @@
 #!/bin/bash
 set -e
 
-# The name and location of the br2-external tree, once validated.
-declare BR2_NAME
-declare BR2_EXT
+# The names and locations of the br2-external trees, once validated.
+declare -a BR2_EXT_NAMES
+declare -A BR2_EXT_PATHS
 
 main() {
     local OPT OPTARG

@@ -22,8 +22,6 @@ main() {
     # Forget options; keep only positional args
     shift $((OPTIND-1))
 
-    br2_ext="${1}"
-
     case "${ofmt}" in
     mk)
         error() {

@@ -43,13 +41,14 @@ main() {
 
     exec >"${ofile}"
 

support/scripts/br2-external +70 -28

+# and BR2_EXT_PATHS.
 #
 # Note: since this script is always first called from Makefile context
 # to generate the Makefile fragment before it is called to generate the

@@ -58,14 +57,22 @@ main() {
 # snippet means that there were no error.
 #
 do_validate() {
+    local br2_ext
+
+    if [ ${#} -eq 0 ]; then
+        # No br2-external tree is valid
+        return
+    fi
+
+    for br2_ext in "${@}"; do
+        do_validate_one "${br2_ext}"
+    done
+}
+
+do_validate_one() {
     local br2_ext="${1}"
     local br2_name n
 
-    # No br2-external tree is valid
-    if [ -z "${br2_ext}" ]; then
-        return
-    fi
-
     if [ ! -d "${br2_ext}" ]; then
         error "'%s': no such file or directory\n" "${br2_ext}"
     fi

@@ -83,6 +90,10 @@ do_validate() {
         error "'%s': name '%s' contains invalid chars: '%s'\n" \
             "${br2_ext}" "${br2_name//\$/\$\$}" "${n//\$/\$\$}"
     fi
+    if [ -n "${BR2_EXT_PATHS["${br2_name}"]}" ]; then
+        error "'%s': name '%s' is already used in '%s'\n" \
+            "${br2_ext}" "${br2_name}" "${BR2_EXT_PATHS["${br2_name}"]}"
+    fi
     if [ ! -f "${br2_ext}/external.mk" ]; then
         error "'%s/external.mk': no such file or directory\n" "${br2_ext}"
     fi

@@ -90,51 +101,80 @@ do_validate() {
         error "'%s/Config.in': no such file or directory\n" "${br2_ext}"
     fi
 
-    BR2_NAME="${br2_name}"
-    BR2_EXT="$(cd "${br2_ext}"; pwd -P )"
+    # Register this br2-external tree
+    BR2_EXT_NAMES+=( "${br2_name}" )
+    BR2_EXT_PATHS["${br2_name}"]="${br2_ext}"
 }
 
 # Generate the .mk snippet that defines makefile variables
 # for the br2-external tree
 do_mk() {
+    local br2_name br2_ext
+
     printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
     printf '\n'
 
-    printf 'BR2_EXTERNAL ?= %s\n' "${BR2_EXT}"
-    printf 'BR2_EXTERNAL_NAME = \n'
-    printf 'BR2_EXTERNAL_MK =\n'
+    # We can't use ${BR2_EXT_NAMES[@]} directly: it is not guaranteed
+    # to be in the order paths were added (because it is an associative
+    # array). So we need to iterate on BR2_EXT_NAMES, which is sorted
+    # in the order names were added (because it is an indexed array).
+    printf 'BR2_EXTERNAL ?='
+    for br2_name in "${BR2_EXT_NAMES[@]}"; do
+        printf ' %s' "${BR2_EXT_PATHS["${br2_name}"]}"
+    done
     printf '\n'
 
-    if [ -z "${BR2_NAME}" ]; then
+    printf 'BR2_EXTERNAL_NAMES = \n'
+    printf 'BR2_EXTERNAL_DIRS = \n'
+    printf 'BR2_EXTERNAL_MKS = \n'
+
+    if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then
+        printf '\n'
         printf '# No br2-external tree defined.\n'
         return
     fi
 
-    printf 'BR2_EXTERNAL_NAME = %s\n' "${BR2_NAME}"
-    printf 'BR2_EXTERNAL_MK = %s/external.mk\n' "${BR2_EXT}"
-    printf 'BR2_EXTERNAL_%s_PATH = %s\n' "${BR2_NAME}" "${BR2_EXT}"
+    for br2_name in "${BR2_EXT_NAMES[@]}"; do
+        br2_ext="${BR2_EXT_PATHS["${br2_name}"]}"
+        printf '\n'
+        printf 'BR2_EXTERNAL_NAMES += %s\n' "${br2_name}"
+        printf 'BR2_EXTERNAL_%s_PATH = %s\n' "${br2_name}" "${br2_ext}"
+        printf 'BR2_EXTERNAL_DIRS += %s\n' "${br2_ext}"
+        printf 'BR2_EXTERNAL_MKS += %s/external.mk\n' "${br2_ext}"
+    done
 }
 
 # Generate the kconfig snippet for the br2-external tree.
 do_kconfig() {
+    local br2_name br2_ext
+
     printf '#\n# Automatically generated file; DO NOT EDIT.\n#\n'
     printf '\n'
 
-    if [ -z "${BR2_NAME}" ]; then
+    if [ ${#BR2_EXT_NAMES[@]} -eq 0 ]; then
         printf '# No br2-external tree defined.\n'
         return
     fi
 
-    printf 'menu "User-provided options"\n'
-    printf '\n'
-    printf 'comment "%s (in %s)"\n' "${BR2_NAME}" "${BR2_EXT}"
-    printf '\n'
-    printf 'config BR2_EXTERNAL_%s_PATH\n' "${BR2_NAME}"
-    printf '\tstring\n'
-    printf '\tdefault "%s"\n' "${BR2_EXT}"
-    printf '\n'
-    printf 'source "$BR2_EXTERNAL_%s_PATH/Config.in"\n' "${BR2_NAME}"
+    printf 'menu "External options"\n'
     printf '\n'
+
+    for br2_name in "${BR2_EXT_NAMES[@]}"; do
+        br2_ext="${BR2_EXT_PATHS["${br2_name}"]}"
+        if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then
+            printf 'menu "%s"\n' "${br2_name}"
+        fi
+        printf 'comment "%s (in %s)"\n' "${br2_name}" "${br2_ext}"
+        printf 'config BR2_EXTERNAL_%s_PATH\n' "${br2_name}"
+        printf '\tstring\n'
+        printf '\tdefault "%s"\n' "${br2_ext}"
+        printf 'source "%s/Config.in"\n' "${br2_ext}"
+        if [ ${#BR2_EXT_NAMES[@]} -gt 1 ]; then
+            printf 'endmenu # %s\n' "${br2_name}"
+        fi
+        printf '\n'
+    done
+
     printf "endmenu # User-provided options\n"
 }
 

@@ -144,12 +184,12 @@ help() {
 	    ${my_name} <-m|-k> -o FILE PATH
 
 	With -m, ${my_name} generates the makefile fragment that defines
-	variables related to the br2-external tree passed as positional
-	argument.
+	variables related to the br2-external trees passed as positional
+	arguments.
 
 	With -k, ${my_name} generates the kconfig snippet to include the
-	configuration options specified in the br2-external tree passed
-	as positional argument.
+	configuration options specified in the br2-external trees passed
+	as positional arguments.
 
 	Using -k and -m together is not possible. The last one wins.
 

Signed-off-by "Yann E. MORIN" <[email protected]>

Cc Thomas Petazzoni <[email protected]>

Cc Arnout Vandecappelle <[email protected]>

Cc Samuel Martin <[email protected]>

Cc Romain Naour <[email protected]>

Cc Julien CORJON <[email protected]>

---
 docs/manual/appendix.txt                |  2 +-
 docs/manual/br2-external-converting.txt | 39 +++++++++++++++++++++++++++++++++
 docs/manual/customize-outside-br.txt    |  5 +++++
 support/scripts/br2-external            |  7 +++++-
 4 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 docs/manual/br2-external-converting.txt

docs/manual/appendix.txt +1 -1

index 87a20bd..cd97744 100644
@@ -3,7 +3,7 @@
 
 include::makedev-syntax.txt[]
 include::makeusers-syntax.txt[]
-
+include::br2-external-converting.txt[]
 
 // Automatically generated lists:
 

docs/manual/br2-external-converting.txt new file +39 -0

new file mode 100644
index 0000000..444c73d
@@ -0,0 +1,39 @@
+// -*- mode:doc; -*-
+// vim: set syntax=asciidoc:
+
+[[br2-external-converting]]
+== Converting old br2-external trees
+
+Before Buildroot 2016.11, it was possible to use only one br2-external
+tree at once. With Buildroot 2016.11 came the possibility to use more
+than one (for details, see xref:outside-br-custom[]).
+
+This however means that older br2-external trees are not useable as-is.
+A minor change has to be made: adding a name to your br2-external tree.
+
+This can be done very easily in just a few steps:
+
+ * First, create a new file named +external.desc+, at the root of your
+   br2-external tree, with a single line defining the name of your
+   br2-external tree:
++
+----
+$ echo 'name: NAME_OF_YOUR_TREE' >external.desc
+----
++
+.Note
+Be careful when choosing a name: it has to be relatively unique, be made
+with only ASCII characters from the set +[A-Za-z0-9_]+.
+
+ * Then, change every occurence of +BR2_EXTERNAL+ in your br2-external
+   tree with the new value:
++
+----
+$ find . -type f | xargs sed -i 's/BR2_EXTERNAL/BR2_EXTERNAL_NAME_OF_YOUR_TREE_PATH/g'
+----
+
+Now, your br2-external tree can be used with Buildroot 2016.11 onward.
+
+.Note:
+This is change makes your br2-external tree incompatible with Buildroot
+before 2016.11.

docs/manual/customize-outside-br.txt +5 -0

index b0c7271..c4f6023 100644
@@ -33,6 +33,11 @@ If it is passed as a relative path, it is important to note that it is
 interpreted relative to the main Buildroot source directory, *not* to
 the Buildroot output directory.
 
+.Note:
+If using an br2-external tree from before Buildroot 2016.11, you need to
+convert it before you can use it with Buildroot 2016.11 onward. See
+xref:br2-external-converting[] for help on doing so.
+
 Some examples:
 
 -----

support/scripts/br2-external +6 -1

index 9936650..612ae58 100755
@@ -5,6 +5,10 @@ set -e
 declare -a BR2_EXT_NAMES
 declare -A BR2_EXT_PATHS
 
+# URL to manual for help in converting old br2-external trees.
+# Escape '#' so that make does not consider it a comment.
+MANUAL_URL='https://buildroot.org/manual/manual.html\#br2-external-converting'
+
 main() {
     local OPT OPTARG
     local br2_ext ofile ofmt

@@ -77,7 +81,8 @@ do_validate_one() {
         error "'%s': no such file or directory\n" "${br2_ext}"
     fi
     if [ ! -f "${br2_ext}/external.desc" ]; then
-        error "'%s': does not have a name (in 'external.desc')\n" "${br2_ext}"
+        error "'%s': does not have a name (in 'external.desc'). See %s\n" \
+            "${br2_ext}" "${MANUAL_URL}"
     fi
     br2_name="$(sed -r -e '/^name: +(.*)$/!d; s//\1/' "${br2_ext}/external.desc")"
     if [ -z "${br2_name}" ]; then

I think the idea of this patch (document how to convert old br2-external tree) should be moved between patches 6 & 7 or merged in patch "6 - docs/manual: document the br2-external NAME"

This rather complex series introduces support for using multiple br2-external trees at once.

This series is an attempt at making it possible to use such setups.