fetch2/crate: support configurable registry and index URLs
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH v2 1/1 fetch2/crate: support configurable registry and index URLs
The crate fetcher previously hardcoded the crates.io download URL and only supported a fixed /crate/versions API shape for custom registries. This prevented recipes from using private registries or mirrors with different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates with {crate}, {version}, and {index_path} placeholders. Keep the existing crates.io defaults, allow custom API-style version endpoints, and mark sparse indexes explicitly so latest-version checks parse Cargo index NDJSON instead of JSON API responses.
Add non-network tests for default crates.io URLs, custom registry templates, sparse index paths, trailing slash handling, and both latest-version parser paths. Also make the fetch test cleanup chmod invocation portable by placing -R before the mode.
[YOCTO #16276]
--- lib/bb/fetch2/crate.py | 39 +++++++++++-- lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 5 deletions(-)
lib/bb/fetch2/crate.py +35 -4
@@ -77,12 +77,43 @@ class Crate(Wget): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # If using crates.io use the CDN directly as per https://crates.io/data-access + # Allow overriding the registry and index URLs via bitbake variables. + # Example: + # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download" + # BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}" + dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host) + index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host) + if host == 'crates.io': - ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) - ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + if not dl_url: + dl_url = "https://static.crates.io/crates/{crate}/{version}/download" + if not index_url: + index_url = "https://index.crates.io/" + + if dl_url: + ud.url = dl_url.replace('{crate}', name).replace('{version}', version) else: ud.url = "https://%s/%s/%s/download" % (host, name, version) + + ud.crate_index_format = 'api' + if index_url: + index_path = self._generate_index_path(name) + # Support {index_path} for sparse index registries. If not present + # and it's crates.io, append it for backward compatibility. + if '{index_path}' in index_url: + ud.versionsurl = index_url.replace('{index_path}', index_path) + ud.crate_index_format = 'sparse' + elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'): + if not index_url.endswith('/'): + index_url += '/' + ud.versionsurl = index_url + index_path + ud.crate_index_format = 'sparse' + else: + ud.versionsurl = index_url + + # Apply other placeholders + ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version) + else: ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -160,7 +191,7 @@ class Crate(Wget): Return the latest upstream version, dispatching to the appropriate parser based on the versionsurl format. """ - if ud.versionsurl.startswith('https://index.crates.io/'): + if getattr(ud, 'crate_index_format', None) == 'sparse': return self._latest_versionstring_from_index(ud, d) return self._latest_versionstring_from_api(ud, d)
lib/bb/tests/fetch.py +121 -1
134 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)
Quoted 16 lines · minsung.cho via lists.openembedded.org
The crate fetcher previously hardcoded the crates.io download URL and only supported a fixed /crate/versions API shape for custom registries. This prevented recipes from using private registries or mirrors with different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates with {crate}, {version}, and {
index_path} placeholders. Keep the existing crates.io defaults, allow custom API-style version endpoints, and mark sparse indexes explicitly so latest-version checks parse Cargo index NDJSON instead of JSON API responses.Add non-network tests for default crates.io URLs, custom registry templates, sparse index paths, trailing slash handling, and both latest-version parser paths. Also make the fetch test cleanup chmod invocation portable by placing -R before the mode.
Quoted 4 lines
[YOCTO #16276]
Signed-off-by minsung.cho <[email protected]>---
Hi,
Thanks for your patch.
I note it does conflict with some other patches currently in testing, and once these conflicts solved, I get some bitbake-selftest errors:
ERROR: test_crate_fetches_from_local_sparse_registry (bb.tests.fetch.CrateTest.test_crate_fetches_from_local_sparse_registry)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/tests/fetch.py", line 2829, in test_crate_fetches_from_local_sparse_registry
self.assertEqual(ud.method.latest_versionstring(ud, self.d),
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 197, in latest_versionstring
return self._latest_versionstring_from_api(ud, d)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 204, in _latest_versionstring_from_api
versions = [(0, i["num"], "") for i in json_data["versions"]]
~~~~~~~~~^^^^^^^^^^^^KeyError: 'versions'
https://autobuilder.yoctoproject.org/valkyrie/#/builders/35/builds/3864
https://autobuilder.yoctoproject.org/valkyrie/#/builders/48/builds/3727
https://autobuilder.yoctoproject.org/valkyrie/#/builders/23/builds/3960
Maybe the easiest solution is to wait a few days until said patches are either merged or rejected, then send a refreshed version.
If you want to have a look at the current situation, here is the branch that was used for above builds:
https://git.yoctoproject.org/poky-ci-archive/log/?h=bitbake/autobuilder.yoctoproject.org/valkyrie/a-full-3832
Thanks,
Mathieu
Signature · 3 lines
Mathieu Dubois-Briand, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
Hi Mathieu,
Thanks for testing this and digging into the failure.
I reproduced it locally with master + Chen Qi's filter_regex series + this patch. The KeyError comes from the conflict resolution in latest_versionstring(), not from either patch on its own. The merged tree kept the old prefix check (ud.versionsurl.startswith('
https://index.crates.io/')), but this patch intentionally drops that in favour of an explicit ud.crate_index_format == 'sparse' check, since the whole point is to support custom sparse registries whose index URL isn't on index.crates.io. The local sparse registry test serves its index from 127.0.0.1, so the prefix check sends NDJSON to the JSON API parser.
The right merge keeps the format check and threads filter_regex through both helpers:
def latest_versionstring(self, ud, d, filter_regex=None):
if getattr(ud, 'crate_index_format', None) == 'sparse':
return self._latest_versionstring_from_index(ud, d, filter_regex) return self._latest_versionstring_from_api(ud, d, filter_regex)
With that, CrateTest passes here (12 tests, 5 skipped), including test_crate_fetches_from_local_sparse_registry.
I've got a v3 ready with exactly that resolution, rebased on the filter_regex series. Agreed it's cleaner to wait until that series lands or gets dropped, then I'll send the refreshed v3. I'll watch the thread.
Thanks,
Minsung
Quoted 72 lines · Mathieu Dubois-Briand
Quoted 16 lines · minsung.cho via lists.openembedded.org
The crate fetcher previously hardcoded the crates.io download URL and only supported a fixed /crate/versions API shape for custom registries. This prevented recipes from using private registries or mirrors with different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates with {crate}, {version}, and {
index_path} placeholders. Keep the existing crates.io defaults, allow custom API-style version endpoints, and mark sparse indexes explicitly so latest-version checks parse Cargo index NDJSON instead of JSON API responses.Add non-network tests for default crates.io URLs, custom registry templates, sparse index paths, trailing slash handling, and both latest-version parser paths. Also make the fetch test cleanup chmod invocation portable by placing -R before the mode.
Quoted 4 lines
[YOCTO #16276]
Signed-off-by minsung.cho <[email protected]>---
Hi,
Thanks for your patch.
I note it does conflict with some other patches currently in testing, and once these conflicts solved, I get some bitbake-selftest errors:
ERROR:
test_crate_fetches_from_local_sparse_registry(bb.tests.fetch.CrateTest.test_crate_fetches_from_local_sparse_registry)
----------------------------------------------------------------------
Traceback (most recent call last):
File
"/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/tests/fetch.py", line 2829, intest_crate_fetches_from_local_sparse_registryself.assertEqual(ud.method.latest_versionstring(ud, self.d), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^File
"/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 197, inlatest_versionstring
returnself._latest_versionstring_from_api(ud, d)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/srv/pokybuild/yocto-worker/oe-selftest-debian/build/layers/bitbake/lib/bb/fetch2/crate.py", line 204, in_latest_versionstring_from_apiversions = [(0, i["num"], "") for i in json_data["versions"]] ~~~~~~~~~^^^^^^^^^^^^KeyError: 'versions'
https://autobuilder.yoctoproject.org/valkyrie/#/builders/35/builds/3864
https://autobuilder.yoctoproject.org/valkyrie/#/builders/48/builds/3727
https://autobuilder.yoctoproject.org/valkyrie/#/builders/23/builds/3960Maybe the easiest solution is to wait a few days until said patches are either merged or rejected, then send a refreshed version.
If you want to have a look at the current situation, here is the branch that was used for above builds:
Thanks,
Mathieu--
Mathieu Dubois-Briand, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
PATCH v3 1/1 fetch2/crate: support configurable registry and index URLs
In reply to [email protected]The crate fetcher previously hardcoded the crates.io download URL and only supported a fixed /crate/versions API shape for custom registries. This prevented recipes from using private registries or mirrors with different download paths or Cargo sparse indexes.
Add BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host] templates with {crate}, {version}, and {index_path} placeholders. Keep the existing crates.io defaults, allow custom API-style version endpoints, and mark sparse indexes explicitly so latest-version checks parse Cargo index NDJSON instead of JSON API responses.
Add non-network tests for default crates.io URLs, custom registry templates, sparse index paths, trailing slash handling, and both latest-version parser paths. Also make the fetch test cleanup chmod invocation portable by placing -R before the mode.
[YOCTO #16276]
--- lib/bb/fetch2/crate.py | 39 +++++++++++-- lib/bb/tests/fetch.py | 122 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 5 deletions(-)
lib/bb/fetch2/crate.py +35 -4
@@ -78,12 +78,43 @@ class Crate(Wget): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # If using crates.io use the CDN directly as per https://crates.io/data-access + # Allow overriding the registry and index URLs via bitbake variables. + # Example: + # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host.com/crates/{crate}/{version}/download" + # BB_CRATE_INDEX_URL[my-host] = "https://my-host.com/index/{index_path}" + dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host) + index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host) + if host == 'crates.io': - ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) - ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + if not dl_url: + dl_url = "https://static.crates.io/crates/{crate}/{version}/download" + if not index_url: + index_url = "https://index.crates.io/" + + if dl_url: + ud.url = dl_url.replace('{crate}', name).replace('{version}', version) else: ud.url = "https://%s/%s/%s/download" % (host, name, version) + + ud.crate_index_format = 'api' + if index_url: + index_path = self._generate_index_path(name) + # Support {index_path} for sparse index registries. If not present + # and it's crates.io, append it for backward compatibility. + if '{index_path}' in index_url: + ud.versionsurl = index_url.replace('{index_path}', index_path) + ud.crate_index_format = 'sparse' + elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'): + if not index_url.endswith('/'): + index_url += '/' + ud.versionsurl = index_url + index_path + ud.crate_index_format = 'sparse' + else: + ud.versionsurl = index_url + + # Apply other placeholders + ud.versionsurl = ud.versionsurl.replace('{crate}', name).replace('{version}', version) + else: ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -161,7 +192,7 @@ class Crate(Wget): Return the latest upstream version, dispatching to the appropriate parser based on the versionsurl format. """ - if ud.versionsurl.startswith('https://index.crates.io/'): + if getattr(ud, 'crate_index_format', None) == 'sparse': return self._latest_versionstring_from_index(ud, d, filter_regex) return self._latest_versionstring_from_api(ud, d, filter_regex)
lib/bb/tests/fetch.py +121 -1
134 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)
Hi,
lib/bb/fetch2/crate.py class Crate(Wget): · line 97
Quoted 4 lines · minsung.cho via lists.openembedded.org
lib/bb/fetch2/crate.py +2
-0+ if dl_url: + ud.url = dl_url.replace('{crate}', name).replace('{version}', version) else: ud.url = "https://%s/%s/%s/download" % (host, name, version)
Instead of using replace() calls, we can treat the string that looks exactly like a f-string as an f-string, and use format(). eg:
ud.url = dl_url.format(version=version, crate=name)
lib/bb/fetch2/crate.py class Crate(Wget): · line 111
Quoted 5 lines
lib/bb/fetch2/crate.py +5
-0+ elif host == 'crates.io' or index_url.startswith('https://index.crates.io/'): + if not index_url.endswith('/'): + index_url += '/' + ud.versionsurl = index_url + index_path + ud.crate_index_format = 'sparse'
It looks like setting index_url to include {index_path} in the crates.io <http://crates.io/> special-casing would mean this entire block can be deleted, as the block above would handle it?
Quoted 1 line
+ ud.versionsurl =
ud.versionsurl.replace('{crate}', name).replace('{version}', version)
Again, use format().
lib/bb/fetch2/crate.py class Crate(Wget): · line 195
Quoted 2 lines
lib/bb/fetch2/crate.py +1
-1- if ud.versionsurl.startswith('https://index.crates.io/'): + if getattr(ud, 'crate_index_format', None) == 'sparse':
When is crate_index_format not set? If it’s not been set in the codepaths above, the other variables haven’t been set either. I think this can be a normal attribute lookup.
lib/bb/tests/fetch.py class FetcherTest(unittest.TestCase): · line 427
Quoted 2 lines
lib/bb/tests/fetch.py +1
-1- bb.process.run('chmod u+rw -R %s' % self.tempdir) + bb.process.run('chmod -R u+rw %s' % self.tempdir)
Separate commit please, as this is unrelated to crate registry fetching. Also, as you’re changing this command please switch it to use a list instead of a string, that is [‘chmod’, ‘-R’, ‘u+rw’, self.tempdir].
Many thanks for writing an actual test case!
Thanks,
Ross
PATCH v4 1/1 fetch2/crate: support configurable registry and index URLs
In reply to minsung.choThe crate fetcher hardcoded the crates.io download URL and assumed every other host served a fixed /name/versions JSON API. Recipes pointing at a private registry or a Cargo sparse index mirror just did not work.
Two new per-host varflags fix this: BB_CRATE_REGISTRY_URL[host] and BB_CRATE_INDEX_URL[host]. Both take {crate} and {version} placeholders; the index template additionally takes {index_path}, and its presence is how the fetcher knows to treat the response as a Cargo sparse index (NDJSON) rather than as a JSON versions API. The crates.io defaults are expressed with the same templates, so there is no separate code path for that host.
Tests cover the crates.io defaults, custom API and sparse templates, both latest-version parser paths, and an end-to-end fetch+unpack against a local sparse registry served over HTTP.
[YOCTO #16276]
---
v4: address Ross Burton's review of v3
- build the download/index URLs with str.format() instead of chained .replace() calls, as the templates are already f-string shaped
- express the crates.io index default with the {index_path} template too, so the generic path handles it and the crates.io-specific index branch (and its trailing-slash handling) is gone
- latest_versionstring() dispatches on ud.crate_index_format directly instead of a getattr() guard; it is always set in _crate_urldata_init
- dropped the separate test-cleanup chmod tweak: master already converted that call to an argument list
- dropped the test for the removed trailing-slash branch Rebased onto master (resolved against the filter_regex series so latest_versionstring keeps the filter_regex parameter).
lib/bb/fetch2/crate.py | 29 +++++++++-- lib/bb/tests/fetch.py | 112 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 4 deletions(-)
lib/bb/fetch2/crate.py +25 -4
@@ -78,12 +78,33 @@ class Crate(Wget): # host (this is to allow custom crate registries to be specified host = '/'.join(parts[2:-2]) - # If using crates.io use the CDN directly as per https://crates.io/data-access + # Allow overriding the registry and index URLs per host via varflags. + # The templates accept {crate}, {version} and {index_path} placeholders: + # BB_CRATE_REGISTRY_URL[my-host] = "https://my-host/crates/{crate}/{version}/download" + # BB_CRATE_INDEX_URL[my-host] = "https://my-host/index/{index_path}" + # A {index_path} placeholder marks a Cargo sparse index; without it the + # index URL is treated as a JSON versions API endpoint. + dl_url = d.getVarFlag('BB_CRATE_REGISTRY_URL', host) + index_url = d.getVarFlag('BB_CRATE_INDEX_URL', host) + + # crates.io uses the CDN directly as per https://crates.io/data-access if host == 'crates.io': - ud.url = "https://static.crates.io/crates/%s/%s/download" % (name, version) - ud.versionsurl = 'https://index.crates.io/' + self._generate_index_path(name) + if not dl_url: + dl_url = "https://static.crates.io/crates/{crate}/{version}/download" + if not index_url: + index_url = "https://index.crates.io/{index_path}" + + if dl_url: + ud.url = dl_url.format(crate=name, version=version) else: ud.url = "https://%s/%s/%s/download" % (host, name, version) + + if index_url: + ud.crate_index_format = 'sparse' if '{index_path}' in index_url else 'api' + ud.versionsurl = index_url.format(crate=name, version=version, + index_path=self._generate_index_path(name)) + else: + ud.crate_index_format = 'api' ud.versionsurl = "https://%s/%s/versions" % (host, name) ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
@@ -161,7 +182,7 @@ class Crate(Wget): Return the latest upstream version, dispatching to the appropriate parser based on the versionsurl format. """ - if ud.versionsurl.startswith('https://index.crates.io/'): + if ud.crate_index_format == 'sparse': return self._latest_versionstring_from_index(ud, d, filter_regex) return self._latest_versionstring_from_api(ud, d, filter_regex)
lib/bb/tests/fetch.py +112 -0
118 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)