support/download: accept user-provided list of extra hash files (branch yem/multi-hash)

index 04f5241d05..7150018a3b 100644
@@ -534,9 +534,9 @@ typically indicates that the +.hash+ file is wrong but the downloaded
 file is probably OK.
 
 Hashes are currently checked for files fetched from http/ftp servers,
-Git repositories, files copied using scp and local files. Hashes are
-not checked for other version control systems (such as Subversion,
-CVS, etc.) because Buildroot currently does not generate reproducible
+Git or subversion repositories, files copied using scp and local files.
+Hashes are not checked for other version control systems (such as CVS,
+mercurial) because Buildroot currently does not generate reproducible
 tarballs when source code is fetched from such version control
 systems.
 

index 5a47f49bc3..03a6557187 100755
@@ -3,12 +3,12 @@ set -e
 
 # Helper to check a file matches its known hash
 # Call it with:
-#   $1: the path of the file containing all the expected hashes
-#   $2: the full path to the temporary file that was downloaded, and
+#   $1: the full path to the temporary file that was downloaded, and
 #       that is to be checked
-#   $3: the final basename of the file, to which it will be ultimately
+#   $2: the final basename of the file, to which it will be ultimately
 #       saved as, to be able to match it to the corresponding hashes
 #       in the .hash file
+#   $*: the paths of the files containing all the expected hashes
 #
 # Exit codes:
 #   0:  the hash file exists and the file to check matches all its hashes,

@@ -27,28 +27,21 @@ while getopts :q OPT; do
 done
 shift $((OPTIND-1))
 
-h_file="${1}"
-file="${2}"
-base="${3}"
-
-# Bail early if no hash to check
-if [ -z "${h_file}" ]; then
-    exit 0
-fi
-# Does the hash-file exist?
-if [ ! -f "${h_file}" ]; then
-    printf "WARNING: no hash file for %s\n" "${base}" >&2
-    exit 0
-fi
+file="${1}"
+base="${2}"
+shift 2
+declare -a h_files=( "${@}" )
 
 # Check one hash for a file
 # $1: algo hash
 # $2: known hash
 # $3: file (full path)
+# $4: hash file (full path)
 check_one_hash() {
     _h="${1}"
     _known="${2}"
     _file="${3}"
+    _h_file="${4}"
 
     # Note: md5 is supported, but undocumented on purpose.
     # Note: sha3 is not supported, since there is currently no implementation

@@ -70,6 +63,7 @@ check_one_hash() {
         return 0
     fi
 
+    printf "ERROR: while checking hashes from %s\n" "${_h_file}" >&2
     printf "ERROR: %s has wrong %s hash:\n" "${base}" "${_h}" >&2
     printf "ERROR: expected: %s\n" "${_known}" >&2
     printf "ERROR: got     : %s\n" "${_hash}" >&2

@@ -79,21 +73,31 @@ check_one_hash() {
 }
 
 # Do we know one or more hashes for that file?
+nb_h_files=0
 nb_checks=0
-while read t h f; do
-    case "${t}" in
-        ''|'#'*)
-            # Skip comments and empty lines
-            continue
-            ;;
-        *)
-            if [ "${f}" = "${base}" ]; then
-                check_one_hash "${t}" "${h}" "${file}"
-                : $((nb_checks++))
-            fi
-            ;;
-    esac
-done <"${h_file}"
+for h_file in "${h_files[@]}"; do
+    [ -f "${h_file}" ] || continue
+    : $((nb_h_files++))
+    while read t h f; do
+        case "${t}" in
+            ''|'#'*)
+                # Skip comments and empty lines
+                continue
+                ;;
+            *)
+                if [ "${f}" = "${base}" ]; then
+                    check_one_hash "${t}" "${h}" "${file}" "${h_file}"
+                    : $((nb_checks++))
+                fi
+                ;;
+        esac
+    done <"${h_file}"
+done
+
+if [ ${nb_h_files} -eq 0 ]; then
+    printf "WARNING: no hash file for %s\n" "${base}" >&2
+    exit 0
+fi
 
 if [ ${nb_checks} -eq 0 ]; then
     case " ${BR_NO_CHECK_HASH_FOR} " in

index 1e8d6058f6..35428faeef 100755
@@ -21,8 +21,8 @@ export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:lru:qf:e"
 
 main() {
     local OPT OPTARG
-    local backend output hfile large_file recurse quiet rc
-    local -a uris
+    local backend output large_file recurse quiet rc
+    local -a uris hfiles
 
     # Parse our options; anything after '--' is for the backend
     while getopts ":c:d:D:o:n:N:H:lrf:u:qp:" OPT; do

@@ -33,7 +33,7 @@ main() {
         o)  output="${OPTARG}";;
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
-        H)  hfile="${OPTARG}";;
+        H)  hfiles+=( "${OPTARG}" );;
         l)  large_file="-l";;
         r)  recurse="-r";;
         f)  filename="${OPTARG}";;

@@ -70,7 +70,7 @@ main() {
     # - fails at least one of its hashes: force a re-download
     # - there's no hash (but a .hash file): consider it a hard error
     if [ -e "${output}" ]; then
-        if support/download/check-hash ${quiet} "${hfile}" "${output}" "${output##*/}"; then
+        if support/download/check-hash ${quiet} "${output}" "${output##*/}" "${hfiles[@]}"; then
             exit 0
         elif [ ${?} -ne 2 ]; then
             # Do not remove the file, otherwise it might get re-downloaded

@@ -154,7 +154,7 @@ main() {
 
         # Check if the downloaded file is sane, and matches the stored hashes
         # for that file
-        if support/download/check-hash ${quiet} "${hfile}" "${tmpf}" "${output##*/}"; then
+        if support/download/check-hash ${quiet} "${tmpf}" "${output##*/}" "${hfiles[@]}"; then
             rc=0
         else
             if [ ${?} -ne 3 ]; then

index 556b6c2575..a808e51a68 100644
@@ -664,12 +664,12 @@ config BR2_PACKAGE_OVERRIDE_FILE
 	  documentation for more details on this feature.
 
 config BR2_GLOBAL_PATCH_DIR
-	string "global patch directories"
+	string "global patch and hash directories"
 	help
 	  You may specify a space separated list of one or more
-	  directories containing global package patches. For a specific
-	  version <packageversion> of a specific package <packagename>,
-	  patches are applied as follows:
+	  directories containing global package patches and/or hashes.
+	  For a specific version <packageversion> of a specific package
+	  <packagename>, patches are looked up as follows:
 
 	  First, the default Buildroot patch set for the package is
 	  applied from the package's directory in Buildroot.

@@ -683,6 +683,9 @@ config BR2_GLOBAL_PATCH_DIR
 	  exists, then all *.patch files in the directory will be
 	  applied.
 
+	  The hash files are looked up similarly to the patches, with
+	  an additional fallback in <global-patch-dir>/all.hash.
+
 menu "Advanced"
 
 config BR2_FORCE_HOST_BUILD

index 7150018a3b..da7aa4640e 100644
@@ -540,6 +540,12 @@ mercurial) because Buildroot currently does not generate reproducible
 tarballs when source code is fetched from such version control
 systems.
 
+Additionally, for packages for which it is possible to specify a custom
+version (e.g. a custom version string, a remote tarball URL, or a VCS
+repository location and changeset, Buildroot can't carry hashes for
+those. It is however possible to xref:customize-hashes[provide a list of
+extra hashes] that can cover such cases.
+
 Hashes should only be added in +.hash+ files for files that are
 guaranteed to be stable. For example, patches auto-generated by Github
 are not guaranteed to be stable, and therefore their hashes can change

index eb98d1bea2..925ea09d8e 100644
@@ -1,8 +1,10 @@
 // -*- mode:doc -*- ;
 // vim: set syntax=asciidoc:
 
+=== Adding project-specific patches and hashes
+
 [[customize-patches]]
-=== Adding project-specific patches
+==== Providing extra patches
 
 It is sometimes useful to apply 'extra' patches to packages - on top of
 those provided in Buildroot. This might be used to support custom

@@ -57,3 +59,24 @@ are available at a URL. *Note:* +BR2_LINUX_KERNEL_PATCH+ specifies kernel
 patches that are applied after patches available in +BR2_GLOBAL_PATCH_DIR+,
 as it is done from a post-patch hook of the Linux package.
 
+
+[[customize-hashes]]
+==== Providing extra hashes
+
+Buildroot bundles a xref:adding-packages-hash[list of hashes] against
+which it checks the integrity of the downloaded archives, or of those
+it generates locally from VCS checkouts). However, it can only do so
+for the known versions; for packages where it is possible to specify
+a custom version (e.g. a custom version string, a remote tarball URL,
+or a VCS repository location and changeset), Buildroot can't carry
+hashes for those.
+
+For users concerned with the integrity of such downloads, it is possible
+to provide a list of hashes that Buildroot can use to check arbitrary
+downloaded files. Those extra hashes are looked up similarly to the
+extra patches (above); for each directory in +BR2_GLOBAL_PATCH_DIR+,
+the first file to exist is used to check a package download:
+
+* +<global-patch-dir>/<packagename>/<packageversion>/<packagename>.hash+
+* +<global-patch-dir>/<packagename>/<packagename>.hash+
+* +<global-patch-dir>/all.hash+

index e5cd83d859..0d38ff7a5f 100644
@@ -116,6 +116,16 @@ define DOWNLOAD
 		-D '$(DL_DIR)' \
 		-f '$(notdir $(1))' \
 		-H '$($(2)_HASH_FILE)' \
+		$(foreach d, $(call qstrip,$(BR2_GLOBAL_PATCH_DIR)), \
+			-H \
+			$(if $(wildcard $(d)/$($(2)_RAWNAME)/$($(2)_VERSION)/$($(2)_RAWNAME).hash), \
+				'$(d)/$($(2)_RAWNAME)/$($(2)_VERSION)/$($(2)_RAWNAME).hash', \
+				$(if $(wildcard $(d)/$($(2)_RAWNAME)/$($(2)_RAWNAME).hash), \
+					'$(d)/$($(2)_RAWNAME)/$($(2)_RAWNAME).hash', \
+					'$(d)/all.hash' \
+				) \
+			) \
+		) \
 		-n '$($(2)_BASENAME_RAW)' \
 		-N '$($(2)_RAWNAME)' \
 		-o '$($(2)_DL_DIR)/$(notdir $(1))' \

index a808e51a68..7808947a05 100644
@@ -700,6 +700,25 @@ config BR2_FORCE_HOST_BUILD
 
 	  This option will increase build time.
 
+config BR2_DOWNLOAD_FORCE_CHECK_HASHES
+	bool "Force all downloads to have a valid hash"
+	depends on BR2_GLOBAL_PATCH_DIR != ""
+	help
+	  For packages where a custom version or location can be set,
+	  Buildroot does not carry a hash for those custom versions or
+	  locations, so the integrity of such downloads is not verified.
+
+	  Say 'y' here to enforce downloads to have at least one valid
+	  hash (and of course, that all hashes be valid).
+
+	  Those hashes are looked in files in BR2_GLOBAL_PATCH_DIR,
+	  using the same heuristic as for the patches. Additionally, a
+	  file named 'all.hash' can be used to store the hashes of many
+	  packages.
+
+comment "Forcing all downloads to have a valid hash needs a global patch directory"
+	depends on BR2_GLOBAL_PATCH_DIR = ""
+
 config BR2_REPRODUCIBLE
 	bool "Make the build reproducible (experimental)"
 	# SOURCE_DATE_EPOCH support in toolchain-wrapper requires GCC 4.4

index 0d38ff7a5f..a7d856d2c4 100644
@@ -66,9 +66,7 @@ github = https://github.com/$(1)/$(2)/archive/$(3)
 gitlab = https://gitlab.com/$(1)/$(2)/-/archive/$(3)
 
 # Expressly do not check hashes for those files
-# Exported variables default to immediately expanded in some versions of
-# make, but we need it to be recursively-epxanded, so explicitly assign it.
-export BR_NO_CHECK_HASH_FOR =
+BR_NO_CHECK_HASH_FOR =
 
 ################################################################################
 # DOWNLOAD_URIS - List the candidates URIs where to get the package from:

@@ -110,6 +108,7 @@ endif
 define DOWNLOAD
 	$(Q)mkdir -p $($(2)_DL_DIR)
 	$(Q)$(EXTRA_ENV) $($(2)_DL_ENV) \
+	BR_NO_CHECK_HASH_FOR="$(if $(BR2_DOWNLOAD_FORCE_CHECK_HASHES),,$(BR_NO_CHECK_HASH_FOR))" \
 		flock $($(2)_DL_DIR)/.lock $(DL_WRAPPER) \
 		-c '$($(2)_DL_VERSION)' \
 		-d '$($(2)_DL_DIR)' \