support/download: add blake2b-256 hash support
PATCH v3 1/3 support/download: add blake2b-256 hash support
PyPI exposes a blake2b_256 digest for distribution files, and that same digest is used in the hash-based files.pythonhosted.org download paths. Add support for storing those hashes in Buildroot .hash files.
Teach the download hash checker to validate blake2b-256 entries using b2sum -l 256, and update check-package so the new hash type and length are accepted. Also add the CycloneDX hash spelling so generated SBOMs can include blake2b-256 hashes from .hash files.
Update scanpypi to write the PyPI-provided blake2b_256 digest alongside the existing md5 and sha256 entries.
---
Changes v2 -> v3:
- Require b2sum -l 256 in the host coreutils suitability check.
--- docs/manual/adding-packages-directory.adoc | 3 ++- support/dependencies/check-host-coreutils.mk | 2 +- support/dependencies/check-host-coreutils.sh | 3 +++ support/download/check-hash | 7 ++++++- support/testing/tests/utils/test_generate_cyclonedx.py | 9 +++++++++ utils/checkpackagelib/lib_hash.py | 2 +- utils/checkpackagelib/test_lib_hash.py | 4 ++++ utils/generate-cyclonedx | 1 + utils/scanpypi | 7 ++++++- 9 files changed, 33 insertions(+), 5 deletions(-)
docs/manual/adding-packages-directory.adoc +2 -1
@@ -464,7 +464,7 @@ The format of this file is one line for each file for which to check the hash, each line with the following three fields separated by two spaces: * the type of hash, one of: -** +md5+, +sha1+, +sha224+, +sha256+, +sha384+, +sha512+ +** +md5+, +sha1+, +sha224+, +sha256+, +sha384+, +sha512+, +blake2b-256+ * the hash of the file: ** for +md5+, 32 hexadecimal characters ** for +sha1+, 40 hexadecimal characters
@@ -472,6 +472,7 @@ hash, each line with the following three fields separated by two spaces: ** for +sha256+, 64 hexadecimal characters ** for +sha384+, 96 hexadecimal characters ** for +sha512+, 128 hexadecimal characters +** for +blake2b-256+, 64 hexadecimal characters * the name of the file: ** for a source archive: the basename of the file, without any directory component,
support/dependencies/check-host-coreutils.mk +1 -1
@@ -1,5 +1,5 @@ # Check whether the host's coreutils are up to date enough -# to provide 'ln --relative' and 'realpath'. +# to provide 'ln --relative', 'realpath' and 'b2sum -l'. ifeq (,$(call suitable-host-package,coreutils)) BR2_COREUTILS_HOST_DEPENDENCY = host-coreutils
support/dependencies/check-host-coreutils.sh +3 -0
@@ -9,4 +9,7 @@ ln --relative --help >/dev/null 2>&1 # Does realpath exist? realpath --help >/dev/null 2>&1 +# Does b2sum exist and support the BLAKE2b-256 length option? +b2sum -l 256 /dev/null >/dev/null 2>&1 + echo OK
support/download/check-hash +6 -1
@@ -49,6 +49,7 @@ check_one_hash() { case "${_h}" in md5|sha1) ;; sha224|sha256|sha384|sha512) ;; + blake2b-256) ;; *) # Unknown hash, exit with error printf "ERROR: unknown hash '%s' for '%s'\n" \ "${_h}" "${base}" >&2
@@ -57,7 +58,11 @@ check_one_hash() { esac # Do the hashes match? - _hash="$( "${_h}sum" "${_file}" |cut -d ' ' -f 1 )" + if [ "${_h}" = "blake2b-256" ]; then + _hash="$( b2sum -l 256 "${_file}" |cut -d ' ' -f 1 )" + else + _hash="$( "${_h}sum" "${_file}" |cut -d ' ' -f 1 )" + fi if [ "${_hash}" = "${_known}" ]; then printf "%s: OK (%s: %s)\n" "${base}" "${_h}" "${_hash}" return 0
support/testing/tests/utils/test_generate_cyclonedx.py +9 -0
@@ -186,6 +186,7 @@ class TestGenerateCycloneDX(unittest.TestCase): "# source archive checksums\n" "sha256 1111111111111111111111111111111111111111111111111111111111111111 foo-1.2.tar.gz\n" "sha1 2222222222222222222222222222222222222222 foo-1.2.tar.gz\n" + "blake2b-256 3333333333333333333333333333333333333333333333333333333333333333 foo-1.2.tar.gz\n" "sha256 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa LICENSE\n" )
@@ -220,6 +221,10 @@ class TestGenerateCycloneDX(unittest.TestCase): "alg": "SHA-1", "content": "2222222222222222222222222222222222222222", }, + { + "alg": "BLAKE2b-256", + "content": "3333333333333333333333333333333333333333333333333333333333333333", + }, ] }, {
@@ -234,6 +239,10 @@ class TestGenerateCycloneDX(unittest.TestCase): "alg": "SHA-1", "content": "2222222222222222222222222222222222222222", }, + { + "alg": "BLAKE2b-256", + "content": "3333333333333333333333333333333333333333333333333333333333333333", + }, ], } ],
utils/checkpackagelib/lib_hash.py +1 -1
@@ -31,7 +31,7 @@ class HashNumberOfFields(_CheckFunction): class HashType(_CheckFunction): len_of_hash = {"md5": 32, "sha1": 40, "sha224": 56, "sha256": 64, - "sha384": 96, "sha512": 128} + "sha384": 96, "sha512": 128, "blake2b-256": 64} def check_line(self, lineno, text): if _empty_line_or_comment(text):
utils/checkpackagelib/test_lib_hash.py +4 -0
@@ -124,6 +124,10 @@ HashType = [ 'sha512 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' '9012345678\n', []), + ('blake2b-256', + 'any', + 'blake2b-256 1234567890123456789012345678901234567890123456789012345678901234\n', + []), ]
utils/generate-cyclonedx +1 -0
@@ -298,6 +298,7 @@ def cyclonedx_source_hashes(comp, source): "sha256": "SHA-256", "sha512": "SHA-512", "md5": "MD5", + "blake2b-256": "BLAKE2b-256", } hashes = []
utils/scanpypi +6 -1
@@ -672,7 +672,7 @@ class BuildrootPackage(): print('Creating {filename}...'.format(filename=path_to_hash)) lines = [] if self.used_url['digests']['md5'] and self.used_url['digests']['sha256']: - hash_header = '# md5, sha256 from {url}\n'.format( + hash_header = '# md5, sha256, blake2b-256 from {url}\n'.format( url=self.metadata_url) lines.append(hash_header) hash_line = '{method} {digest} {filename}\n'.format(
@@ -685,6 +685,11 @@ class BuildrootPackage(): digest=self.used_url['digests']['sha256'], filename=self.filename) lines.append(hash_line) + hash_line = '{method} {digest} {filename}\n'.format( + method='blake2b-256', + digest=self.used_url['digests']['blake2b_256'], + filename=self.filename) + lines.append(hash_line) if self.license_files: lines.append('# Locally computed sha256 checksums\n')
Signature · 6 lines
2.53.0 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v3 2/3 package/pkg-python: derive PyPI site from blake2b-256 hash
Python packages downloaded from PyPI currently store the complete files.pythonhosted.org package path in their .mk file. That path is hash-based, so it changes for every version bump and adds churn to updates and backports.
When a Python package does not define _SITE, derive the hash-based files.pythonhosted.org site from the blake2b-256 entry matching the package source archive in the package .hash file before the generic package infrastructure validates _SITE.
Follow the usual target/host fallback pattern for _SITE, and raise an explicit error if no hash file is available or if the hash file does not contain a matching blake2b-256 entry. Add the host coreutils dependency for packages using this resolver so b2sum is available when needed for hash verification.
Update scanpypi to omit generated _SITE assignments so newly generated PyPI packages can use the resolver.
---
Changes v2 -> v3:
- Add the host coreutils dependency for packages using the hash-derived PyPI site.
Changes v1 -> v2:
- Follow the usual target/host _SITE fallback pattern.
- Drop the /dev/null awk input and guard missing hash files explicitly.
- Error out when no matching blake2b-256 entry is found.
--- docs/manual/adding-packages-python.adoc | 5 +++++ package/pkg-python.mk | 15 +++++++++++++++ utils/scanpypi | 9 --------- 3 files changed, 20 insertions(+), 9 deletions(-)
docs/manual/adding-packages-python.adoc +5 -0
@@ -82,6 +82,11 @@ All the package metadata information variables that exist in the xref:generic-package-reference[generic package infrastructure] also exist in the Python infrastructure. +For Python packages downloaded from PyPI, the +PYTHON_FOO_SITE+ variable +can be omitted when the package +.hash+ file contains a +blake2b-256+ +hash for the source archive. Buildroot derives the hash-based +files.pythonhosted.org URL from that hash. + Note that: * It is not necessary to add +python+ or +host-python+ in the
package/pkg-python.mk +15 -0
@@ -454,6 +454,21 @@ endif endif # host / target +ifndef $(2)_SITE + ifdef $(3)_SITE + $(2)_SITE = $$($(3)_SITE) + else + $(2)_DOWNLOAD_DEPENDENCIES += $(BR2_COREUTILS_HOST_DEPENDENCY) + $(2)_SITE = $$(strip $$(if $$(strip $$($(2)_HASH_FILES)),\ + $$(or $$(shell \ + awk -v filename="$$(notdir $$($(2)_SOURCE))" \ + '$$$$1 == "blake2b-256" && $$$$3 == filename { h = $$$$2; printf "https://files.pythonhosted.org/packages/%s/%s/%s", substr(h, 1, 2), substr(h, 3, 2), substr(h, 5); exit }' \ + $$($(2)_HASH_FILES)),\ + $$(error $(2)_SITE must be set or $$($(2)_HASH_FILES) must contain a blake2b-256 hash for $$($(2)_SOURCE))),\ + $$(error $(2)_SITE must be set or a hash file must exist to derive the PyPI site))) + endif +endif + # Call the generic package infrastructure to generate the necessary # make targets $(call inner-generic-package,$(1),$(2),$(3),$(4))
utils/scanpypi +0 -9
@@ -495,15 +495,6 @@ class BuildrootPackage(): filename=targz) lines.append(targz_line) - if self.filename not in self.url: - # Sometimes the filename is in the url, sometimes it's not - site_url = self.url - else: - site_url = self.url[:self.url.find(self.filename)] - site_line = '{name}_SITE = {url}'.format(name=self.mk_name, - url=site_url) - site_line = site_line.rstrip('/') + '\n' - lines.append(site_line) return lines def __create_mk_setup(self):
Signature · 6 lines
2.53.0 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v3 2/3] package/pkg-python: derive PyPI site from blake2b-256 hash
Hello,
Quoted 16 lines · James Hilliard
Python packages downloaded from PyPI currently store the complete files.pythonhosted.org package path in their .mk file. That path is hash-based, so it changes for every version bump and adds churn to updates and backports.
When a Python package does not define
_SITE, derive the hash-based files.pythonhosted.org site from the blake2b-256 entry matching the package source archive in the package .hash file before the generic package infrastructure validates_SITE.Follow the usual target/host fallback pattern for
_SITE, and raise an explicit error if no hash file is available or if the hash file does not contain a matching blake2b-256 entry. Add the host coreutils dependency for packages using this resolver so b2sum is available when needed for hash verification.
Quoted 4 lines
Update scanpypi to omit generated
_SITEassignments so newly generated PyPI packages can use the resolver.Signed-off-by James Hilliard <[email protected]>
I must say I am not a fan of this approach. It is very "tricky" and not obvious to understand that the _SITE information is in some cases derived from information in the .hash file. For me this breaks the principle of "obviousness" that makes Buildroot generally quite nice to work with. I don't really find the current situation problematic, and what we have currently is completely obvious to use/understand.
Of course, if the other maintainers are happy with your approach, I will not oppose. But since so far nobody provided feedback, I thought I should share my humble feedback.
Another concern: I assume you're adding $(BR2_COREUTILS_HOST_DEPENDENCY) to DOWNLOAD_DEPENDENCIES so that 'awk' is available? But then if we run "make show-info" without doing any build, it means we no longer have the _SITE information?
Best regards,
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v3 2/3] package/pkg-python: derive PyPI site from blake2b-256 hash
Quoted 23 lines · Thomas Petazzoni
Hello,
Quoted 19 lines · James Hilliard
Python packages downloaded from PyPI currently store the complete files.pythonhosted.org package path in their .mk file. That path is hash-based, so it changes for every version bump and adds churn to updates and backports.
When a Python package does not define
_SITE, derive the hash-based files.pythonhosted.org site from the blake2b-256 entry matching the package source archive in the package .hash file before the generic package infrastructure validates_SITE.Follow the usual target/host fallback pattern for
_SITE, and raise an explicit error if no hash file is available or if the hash file does not contain a matching blake2b-256 entry. Add the host coreutils dependency for packages using this resolver so b2sum is available when needed for hash verification.Update scanpypi to omit generated
_SITEassignments so newly generated PyPI packages can use the resolver.
Quoted 8 lines
Quoted 1 line
Signed-off-by James Hilliard <[email protected]>I must say I am not a fan of this approach. It is very "tricky" and not obvious to understand that the
_SITEinformation is in some cases derived from information in the .hash file. For me this breaks the principle of "obviousness" that makes Buildroot generally quite nice to work with. I don't really find the current situation problematic, and what we have currently is completely obvious to use/understand.
Yeah, it's a bit weird deriving from a non-make file, although auto deriving _SITE in general does have precedent and if the hash is missing there should at least be a clear error explaining the reason:
https://github.com/buildroot/buildroot/blob/2026.08/package/pkg-luarocks.mk#L79-L85
Quoted 3 lines
Of course, if the other maintainers are happy with your approach, I will not oppose. But since so far nobody provided feedback, I thought I should share my humble feedback.
Here's some background on where the suggestion for this change came from:
https://lore.kernel.org/buildroot/[email protected]/
Quoted 3 lines
Another concern: I assume you're adding $(BR2_COREUTILS_HOST_DEPENDENCY) to
DOWNLOAD_DEPENDENCIESso that 'awk' is available?
Isn't awk already mandatory?:
https://github.com/buildroot/buildroot/blob/2026.08/support/dependencies/dependencies.sh#L157
Quoted 2 lines
But then if we run "make show-info" without doing any build, it means we no longer have the
_SITEinformation?
That should still work fine, we still set _SITE with this approach, it only changes how/where it gets set.
Quoted 8 lines
Best regards,
Thomas
--
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training
https://bootlin.com
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v3 2/3] package/pkg-python: derive PyPI site from blake2b-256 hash
Hello,
Quoted 11 lines · James Hilliard
Quoted 6 lines
I must say I am not a fan of this approach. It is very "tricky" and not obvious to understand that the
_SITEinformation is in some cases derived from information in the .hash file. For me this breaks the principle of "obviousness" that makes Buildroot generally quite nice to work with. I don't really find the current situation problematic, and what we have currently is completely obvious to use/understand.Yeah, it's a bit weird deriving from a non-make file, although auto deriving
_SITEin general does have precedent and if the hash is missing there should at least be a clear error explaining the reason:
https://github.com/buildroot/buildroot/blob/2026.08/package/pkg-luarocks.mk#L79-L85
This is not at all the same thing: this is a case where we have a default. If nothing is specified, a default is used, but that default value is not magically inferred from the .hash file.
Quoted 6 lines
Quoted 3 lines
Of course, if the other maintainers are happy with your approach, I will not oppose. But since so far nobody provided feedback, I thought I should share my humble feedback.
Here's some background on where the suggestion for this change came from:
https://lore.kernel.org/buildroot/[email protected]/
OK. I still find this "meh". But again, this is just my own opinion/voice, and I will not oppose to this getting merged. It's just that it's been sitting on patchwork for a bit of time with no feedback, so I thought that to help get things move forward, I would share my feedback.
Quoted 6 lines
Quoted 3 lines
Another concern: I assume you're adding $(BR2_COREUTILS_HOST_DEPENDENCY) to
DOWNLOAD_DEPENDENCIESso that 'awk' is available?Isn't awk already mandatory?:
https://github.com/buildroot/buildroot/blob/2026.08/support/dependencies/dependencies.sh#L157
Indeed. But then why does your patch have:
$(2)_DOWNLOAD_DEPENDENCIES += $(BR2_COREUTILS_HOST_DEPENDENCY)
Quoted 5 lines
Quoted 2 lines
But then if we run "make show-info" without doing any build, it means we no longer have the
_SITEinformation?That should still work fine, we still set
_SITEwith this approach, it only changes how/where it gets set.
Not if $(BR2_COREUTILS_HOST_DEPENDENCY) must be built to calculate the _SITE value (see above question).
Thanks!
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v3 2/3] package/pkg-python: derive PyPI site from blake2b-256 hash
Quoted 38 lines · Thomas Petazzoni
Hello,
Quoted 11 lines · James Hilliard
Quoted 6 lines
I must say I am not a fan of this approach. It is very "tricky" and not obvious to understand that the
_SITEinformation is in some cases derived from information in the .hash file. For me this breaks the principle of "obviousness" that makes Buildroot generally quite nice to work with. I don't really find the current situation problematic, and what we have currently is completely obvious to use/understand.Yeah, it's a bit weird deriving from a non-make file, although auto deriving
_SITEin general does have precedent and if the hash is missing there should at least be a clear error explaining the reason:
https://github.com/buildroot/buildroot/blob/2026.08/package/pkg-luarocks.mk#L79-L85This is not at all the same thing: this is a case where we have a default. If nothing is specified, a default is used, but that default value is not magically inferred from the .hash file.
Quoted 6 lines
Quoted 3 lines
Of course, if the other maintainers are happy with your approach, I will not oppose. But since so far nobody provided feedback, I thought I should share my humble feedback.
Here's some background on where the suggestion for this change came from:
https://lore.kernel.org/buildroot/[email protected]/OK. I still find this "meh". But again, this is just my own opinion/voice, and I will not oppose to this getting merged. It's just that it's been sitting on patchwork for a bit of time with no feedback, so I thought that to help get things move forward, I would share my feedback.
Quoted 4 lines
Quoted 3 lines
Another concern: I assume you're adding $(BR2_COREUTILS_HOST_DEPENDENCY) to
DOWNLOAD_DEPENDENCIESso that 'awk' is available?
Quoted 6 lines
Quoted 2 lines
Isn't awk already mandatory?:
https://github.com/buildroot/buildroot/blob/2026.08/support/dependencies/dependencies.sh#L157Indeed. But then why does your patch have:
$(2)_DOWNLOAD_DEPENDENCIES += $(BR2_COREUTILS_HOST_DEPENDENCY)
That dependency provides b2sum -l 256, which support/download/check-hash uses to verify the downloaded archive. I added it following Yann's review about supporting hosts with older coreutils.
Quoted 9 lines
Quoted 6 lines
Quoted 3 lines
But then if we run "make show-info" without doing any build, it means we no longer have the
_SITEinformation?That should still work fine, we still set
_SITEwith this approach, it only changes how/where it gets set.Not if $(BR2_COREUTILS_HOST_DEPENDENCY) must be built to calculate the
_SITEvalue (see above question).
Resolving _SITE only uses awk to read the digest already recorded in .hash and format it into the PyPI URL. It does not calculate a hash or invoke b2sum, so it does not require host-coreutils to be built.
Quoted 8 lines
Thanks!
Thomas
--
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training
https://bootlin.com
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Re: [Buildroot] [PATCH v3 2/3] package/pkg-python: derive PyPI site from blake2b-256 hash
Quoted 5 lines · James Hilliard
Quoted 1 line
$(2)_DOWNLOAD_DEPENDENCIES += $(BR2_COREUTILS_HOST_DEPENDENCY)That dependency provides b2sum -l 256, which support/download/check-hash uses to verify the downloaded archive. I added it following Yann's review about supporting hosts with older coreutils.
But then that doesn't work either, because it only adds $(BR2_COREUTILS_HOST_DEPENDENCY) in DOWNLOAD_DEPENDENCIES for python packages that don't specify a _SITE variable.
But the blake2 hash support is completely unrelated to Python: any other package could use it, not necessarily a Python package. Or could be a Python package that specifies _SITE explicitly.
Quoted 3 lines
Resolving
_SITEonly uses awk to read the digest already recorded in .hash and format it into the PyPI URL. It does not calculate a hash or invoke b2sum, so it does not require host-coreutils to be built.
Indeed, but see above :-)
(Also, this line adding $(BR2_COREUTILS_HOST_DEPENDENCY) the dependencies would definitely need a comment)
Thomas
Signature · 7 lines
Thomas Petazzoni, co-owner and CEO, Bootlin Embedded Linux and Kernel engineering and training https://bootlin.com _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
PATCH v3 3/3 package/python-async-timeout: use hash-based PyPI site
In reply to James HilliardAdd the blake2b-256 digest from the PyPI metadata and drop the explicit PYTHON_ASYNC_TIMEOUT_SITE assignment.
The Python package infrastructure can now derive the hash-based files.pythonhosted.org site from the package .hash file.
--- package/python-async-timeout/python-async-timeout.hash | 3 ++- package/python-async-timeout/python-async-timeout.mk | 1 - 2 files changed, 2 insertions(+), 2 deletions(-)
package/python-async-timeout/python-async-timeout.hash +2 -1
@@ -1,5 +1,6 @@ -# md5, sha256 from https://pypi.org/pypi/async-timeout/json +# md5, sha256, blake2b-256 from https://pypi.org/pypi/async-timeout/json md5 566a39011e87cb8044ee75369e69b327 async_timeout-5.0.1.tar.gz sha256 d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3 async_timeout-5.0.1.tar.gz +blake2b-256 a5ae136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780 async_timeout-5.0.1.tar.gz # Locally computed sha256 checksums sha256 e18d7bb8f513e2c46bb585c94b585bd30720dd3ccb21ddb0786f72d16658f92c LICENSE
package/python-async-timeout/python-async-timeout.mk +0 -1
@@ -6,7 +6,6 @@ PYTHON_ASYNC_TIMEOUT_VERSION = 5.0.1 PYTHON_ASYNC_TIMEOUT_SOURCE = async_timeout-$(PYTHON_ASYNC_TIMEOUT_VERSION).tar.gz -PYTHON_ASYNC_TIMEOUT_SITE = https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780 PYTHON_ASYNC_TIMEOUT_LICENSE = Apache-2.0 PYTHON_ASYNC_TIMEOUT_LICENSE_FILES = LICENSE PYTHON_ASYNC_TIMEOUT_SETUP_TYPE = setuptools
Signature · 6 lines
2.53.0 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot