fetch2/crate: Simplify extraction of crate names and versions from URIs
PATCH 1/2 fetch2/crate: Simplify extraction of crate names and versions from URIs
---
@Steve Sakoman: Since this is a clean up patch, I will leave it up to you to decide if you want to take it for Mickledore once it makes it into master.
bitbake/lib/bb/fetch2/crate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
bitbake/lib/bb/fetch2/crate.py +3 -3
@@ -59,11 +59,11 @@ class Crate(Wget): # version is expected to be the last token # but ignore possible url parameters which will be used # by the top fetcher class - version, _, _ = parts[len(parts) -1].partition(";") + version = parts[-1].split(";")[0] # second to last field is name - name = parts[len(parts) - 2] + name = parts[-2] # host (this is to allow custom crate registries to be specified - host = '/'.join(parts[2:len(parts) - 2]) + host = '/'.join(parts[2:-2]) # if using upstream just fix it up nicely if host == 'crates.io':
PATCH 2/2 fetch2/crate: Correct unpack for a crate that matches the recipe name
The crate fetcher handles a crate with a name that matches the recipe's name specially by placing the unpacked code in the current directory (which typically is ${S}) rather than together with the sources for the other crates. This broke when the URI names for all crates were changed recently to include the version in the name.
Correct the crate fetcher to test against ${BP} instead of ${BPN}. Also add a test case to the selftests to avoid this breaking again.
--- bitbake/lib/bb/fetch2/crate.py | 4 ++-- bitbake/lib/bb/tests/fetch.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/crate.py +2 -2
@@ -98,8 +98,8 @@ class Crate(Wget): save_cwd = os.getcwd() os.chdir(rootdir) - pn = d.getVar('BPN') - if pn == ud.parm.get('name'): + bp = d.getVar('BP') + if bp == ud.parm.get('name'): cmd = "tar -xz --no-same-owner -f %s" % thefile else: cargo_bitbake = self._cargo_bitbake_path(rootdir)
bitbake/lib/bb/tests/fetch.py +25 -0
@@ -2391,6 +2391,31 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + @skipIfNoNetwork() + def test_crate_url_matching_recipe(self): + + self.d.setVar('BP', 'glob-0.2.11') + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) + @skipIfNoNetwork() def test_crate_url_params(self):
Re: [bitbake-devel] [master][mickledore][PATCH 2/2] fetch2/crate: Correct unpack for a crate that matches the recipe name
Hello Peter,
Do you think it will solve the issue I mentioned there
<https://bugzilla.yoctoproject.org/show_bug.cgi?id=15012#c4> ?
Quoted 84 lines · Peter Kjellerstedt
The crate fetcher handles a crate with a name that matches the recipe's name specially by placing the unpacked code in the current directory (which typically is ${S}) rather than together with the sources for the other crates. This broke when the URI names for all crates were changed recently to include the version in the name.
Correct the crate fetcher to test against ${BP} instead of ${BPN}. Also add a test case to the selftests to avoid this breaking again.
Signed-off-by Peter Kjellerstedt <[email protected]>--- bitbake/lib/bb/fetch2/crate.py | 4 ++-- bitbake/lib/bb/tests/fetch.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-)bitbake/lib/bb/fetch2/crate.py +2
-2@@ -98,8 +98,8 @@ class Crate(Wget): save_cwd = os.getcwd() os.chdir(rootdir) - pn = d.getVar('BPN') - if pn == ud.parm.get('name'): + bp = d.getVar('BP') + if bp == ud.parm.get('name'): cmd = "tar -xz --no-same-owner -f %s" % thefile else: cargo_bitbake = self._cargo_bitbake_path(rootdir)bitbake/lib/bb/tests/fetch.py +23
-0@@ -2391,6 +2391,31 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + @skipIfNoNetwork() + def test_crate_url_matching_recipe(self): + + self.d.setVar('BP', 'glob-0.2.11') + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")),['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+ self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
+
@skipIfNoNetwork()
deftest_crate_url_params(self):-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#14758):
https://lists.openembedded.org/g/bitbake-devel/message/14758
Mute This Topic: https://lists.openembedded.org/mt/98571257/6213388
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
RE: [bitbake-devel] [master][mickledore][PATCH 2/2] fetch2/crate: Correct unpack for a crate that matches the recipe name
Yes. While the original issue was written about missing checksuming, which was fixed by requiring checksums for the crate URIs in the SRC_URI, the additional concern you raised is fixed by my patch. It restores the support for using a crate as the main component for the recipe. This was in the bbclass all along, it just broke when the names were changed to include the crate versions.
You can actually workaround the problem by setting S = "${CARGO_VENDORING_DIRECTORY}/${BP}", but I believe it is better to restore the intended functionality of the bbclass. It also makes things much more intuitive when using devtool modify for such a recipe since the source is where you expect it to be.
@Alexandre Belloni: Since I assume it will be you that pulls this into testing, feel free to add the following to the commit message:
[Yocto #15012]
//Peter
From: Frédéric Martinsons <[email protected]>
Sent: den 29 april 2023 08:48
To: Peter Kjellerstedt <[email protected]>
Cc: [email protected]
Subject: Re: [bitbake-devel] [master][mickledore][PATCH 2/2] fetch2/crate: Correct unpack for a crate that matches the recipe name
Hello Peter,
Do you think it will solve the issue I mentioned there<https://bugzilla.yoctoproject.org/show_bug.cgi?id=15012#c4> ?
On Sat, 29 Apr 2023 at 03:23, Peter Kjellerstedt <[email protected]<mailto:[email protected]>> wrote:
The crate fetcher handles a crate with a name that matches the recipe's
name specially by placing the unpacked code in the current directory
(which typically is ${S}) rather than together with the sources for the
other crates. This broke when the URI names for all crates were changed
recently to include the version in the name.
Correct the crate fetcher to test against ${BP} instead of ${BPN}. Also add a test case to the selftests to avoid this breaking again.
--- bitbake/lib/bb/fetch2/crate.py | 4 ++-- bitbake/lib/bb/tests/fetch.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-)
bitbake/lib/bb/fetch2/crate.py +2 -2
@@ -98,8 +98,8 @@ class Crate(Wget): save_cwd = os.getcwd() os.chdir(rootdir) - pn = d.getVar('BPN') - if pn == ud.parm.get('name'): + bp = d.getVar('BP') + if bp == ud.parm.get('name'): cmd = "tar -xz --no-same-owner -f %s" % thefile else: cargo_bitbake = self._cargo_bitbake_path(rootdir)
bitbake/lib/bb/tests/fetch.py +25 -0
@@ -2391,6 +2391,31 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs<http://lib.rs>")) + @skipIfNoNetwork() + def test_crate_url_matching_recipe(self): + + self.d.setVar('BP', 'glob-0.2.11') + + uri = "crate://crates.io/glob/0.2.11<http://crates.io/glob/0.2.11>" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) + self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs<http://lib.rs>")) + @skipIfNoNetwork() def test_crate_url_params(self):
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#14758): https://lists.openembedded.org/g/bitbake-devel/message/14758
Mute This Topic: https://lists.openembedded.org/mt/98571257/6213388
Group Owner: [email protected]<mailto:bitbake-devel%[email protected]>
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [[email protected]<mailto:[email protected]>]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [bitbake-devel] [master][mickledore][PATCH 2/2] fetch2/crate: Correct unpack for a crate that matches the recipe name
Great, thanks Peter for the confirmation!
Quoted 135 lines · Peter Kjellerstedt
Yes. While the original issue was written about missing checksuming, which was fixed by requiring checksums for the crate URIs in the
SRC_URI, the additional concern you raised is fixed by my patch. It restores the support for using a crate as the main component for the recipe. This was in the bbclass all along, it just broke when the names were changed to include the crate versions.You can actually workaround the problem by setting S = "${CARGO_VENDORING_DIRECTORY}/${BP}", but I believe it is better to restore the intended functionality of the bbclass. It also makes things much more intuitive when using devtool modify for such a recipe since the source is where you expect it to be.
@Alexandre Belloni: Since I assume it will be you that pulls this into testing, feel free to add the following to the commit message:
[Yocto #15012]
//Peter
*From:* Frédéric Martinsons <[email protected]> *Sent:* den 29 april 2023 08:48
*To:* Peter Kjellerstedt <[email protected]> *Cc:* [email protected]
*Subject:* Re: [bitbake-devel] [master][mickledore][PATCH 2/2] fetch2/crate: Correct unpack for a crate that matches the recipe nameHello Peter,
Do you think it will solve the issue I mentioned there
<https://bugzilla.yoctoproject.org/show_bug.cgi?id=15012#c4> ?On Sat, 29 Apr 2023 at 03:23, Peter Kjellerstedt < [email protected]> wrote:
The crate fetcher handles a crate with a name that matches the recipe's name specially by placing the unpacked code in the current directory (which typically is ${S}) rather than together with the sources for the other crates. This broke when the URI names for all crates were changed recently to include the version in the name.
Correct the crate fetcher to test against ${BP} instead of ${BPN}. Also add a test case to the selftests to avoid this breaking again.
Signed-off-by Peter Kjellerstedt <[email protected]>--- bitbake/lib/bb/fetch2/crate.py | 4 ++-- bitbake/lib/bb/tests/fetch.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-)bitbake/lib/bb/fetch2/crate.py +2
-2@@ -98,8 +98,8 @@ class Crate(Wget): save_cwd = os.getcwd() os.chdir(rootdir) - pn = d.getVar('BPN') - if pn == ud.parm.get('name'): + bp = d.getVar('BP') + if bp == ud.parm.get('name'): cmd = "tar -xz --no-same-owner -f %s" % thefile else: cargo_bitbake = self._cargo_bitbake_path(rootdir)bitbake/lib/bb/tests/fetch.py +23
-0@@ -2391,6 +2391,31 @@ class CrateTest(FetcherTest): self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) + @skipIfNoNetwork() + def test_crate_url_matching_recipe(self): + + self.d.setVar('BP', 'glob-0.2.11') + + uri = "crate://crates.io/glob/0.2.11" + self.d.setVar('SRC_URI', uri) + + uris = self.d.getVar('SRC_URI').split() + d = self.d + + fetcher = bb.fetch2.Fetch(uris, self.d) + ud = fetcher.ud[fetcher.urls[0]] + + self.assertIn("name", ud.parm) + self.assertEqual(ud.parm["name"], "glob-0.2.11") + self.assertIn("downloadfilename", ud.parm) + self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") + + fetcher.download() + fetcher.unpack(self.tempdir) + self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) + self.assertEqual(sorted(os.listdir(self.tempdir + "/download")),['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+ self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
+
@skipIfNoNetwork()
deftest_crate_url_params(self):-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group. View/Reply Online (#14758):
https://lists.openembedded.org/g/bitbake-devel/message/14758
Mute This Topic: https://lists.openembedded.org/mt/98571257/6213388
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [ [email protected]]
-=-=-=-=-=-=-=-=-=-=-=-