rpm: add support for remote package feeds via PACKAGE_FEED_URIS variable
PATCH 1/4 rpm: add support for remote package feeds via PACKAGE_FEED_URIS variable
I've used a previous patch (which was never merged) by Humberto Ibarra <[email protected]> as a model for how to do runtime testing of this feature (e.g. we need to boot an image, run dnf on it, and check that it is indeed able to access the remote repo over http). Here's his original commit message:
=====
Testing that feeds specified with PACKAGE_FEED_URIS var are set correctly has two parts. First a build with this var set is required, and then smart update needs to be issued in the running taget.
The previous is not a common selftest practice because this is a simple test, but requires building and running a specific image, which takes a lot of time. testimage is not a good fit either, since the images tested there do not have the PACKAGE_FEED_URIS var set.
For this test, the runtime-test module is being used, which is a selftest module but runs a testimage command. The var and test environment were set in runtime-perf.py and the actual test is done in a new testcase added to meta-selftest layer.
=====
[YOCTO #10872]
--- meta-selftest/lib/oeqa/runtime/cases/selftest.py | 42 ++++++++++++++++++++++++ meta/lib/oe/package_manager.py | 12 ++++++- meta/lib/oeqa/selftest/runtime-test.py | 7 ++++ 3 files changed, 60 insertions(+), 1 deletion(-)
meta-selftest/lib/oeqa/runtime/cases/selftest.py +42 -0
@@ -1,5 +1,7 @@ from oeqa.runtime.case import OERuntimeTestCase from oeqa.core.decorator.depends import OETestDepends +from oeqa.runtime.cases.dnf import DnfTest +from oeqa.utils.httpserver import HTTPService class Selftest(OERuntimeTestCase):
@@ -29,3 +31,43 @@ class Selftest(OERuntimeTestCase): (status, output) = self.target.run("socat -V") self.assertNotEqual(status, 0, msg="socat is still installed") + + +class DnfSelftest(DnfTest): + + @classmethod + def setUpClass(cls): + cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-rootfs-repo'), + cls.tc.target.server_ip) + cls.repo_server.start() + + @classmethod + def tearDownClass(cls): + cls.repo_server.stop() + + @OETestDepends(['ssh.SSHTest.test_ssh']) + def test_verify_package_feeds(self): + """ + Summary: Check correct setting of PACKAGE_FEED_URIS var + Expected: 1. Feeds were correctly set for dnf + 2. Update recovers packages from host's repo + Author: Humberto Ibarra <[email protected]> + Author: Alexander Kanavin <[email protected]> + """ + # When we created an image, we had to supply fake ip and port + # for the feeds. Now we can patch the real ones into the config file. + import tempfile + temp_file = tempfile.TemporaryDirectory(prefix="oeqa-remotefeeds-").name + self.tc.target.copyFrom("/etc/yum.repos.d/oe-remote-repo.repo", temp_file) + fixed_config = open(temp_file, "r").read().replace("bogus_ip", self.tc.target.server_ip).replace("bogus_port", str(self.repo_server.port)) + open(temp_file, "w").write(fixed_config) + self.tc.target.copyTo(temp_file, "/etc/yum.repos.d/oe-remote-repo.repo") + + import re + output_makecache = self.dnf('makecache') + self.assertTrue(re.match(r".*Metadata cache created", output_makecache, re.DOTALL) is not None, msg = "dnf makecache failed: %s" %(output_makecache)) + + output_repoinfo = self.dnf('repoinfo') + matchobj = re.match(r".*Repo-pkgs\s*:\s*(?P<n_pkgs>[0-9]+)", output_repoinfo, re.DOTALL) + self.assertTrue(matchobj is not None, msg = "Could not find the amount of packages in dnf repoinfo output: %s" %(output_repoinfo)) + self.assertTrue(int(matchobj.group('n_pkgs')) > 0, msg = "Amount of remote packages is not more than zero: %s\n" %(output_repoinfo))
meta/lib/oe/package_manager.py +11 -1
@@ -533,10 +533,20 @@ class RpmPM(PackageManager): bb.utils.unlockfile(lf) def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): + from urllib.parse import urlparse + if feed_uris == "": return - raise NotImplementedError("Adding remote dnf feeds not yet supported.") + bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d")) + remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) + for uri in remote_uris: + repo_name = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/")) + if feed_archs is not None: + repo_uris = [uri + "/" + arch for arch in feed_archs] + else: + repo_uris = [uri] + open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_name + ".repo"), 'w').write("[%s]\nbaseurl=%s\n" % (repo_name, " ".join(repo_uris))) def _prepare_pkg_transaction(self): os.environ['D'] = self.target_rootfs
meta/lib/oeqa/selftest/runtime-test.py +7 -0
@@ -108,14 +108,21 @@ class TestImage(oeSelfTest): Summary: Check install packages functionality for testimage/testexport. Expected: 1. Import tests from a directory other than meta. 2. Check install/uninstall of socat. + 3. Check that remote package feeds can be accessed Product: oe-core Author: Mariano Lopez <[email protected]> + Author: Alexander Kanavin <[email protected]> """ if get_bb_var('DISTRO') == 'poky-tiny': self.skipTest('core-image-full-cmdline not buildable for poky-tiny') features = 'INHERIT += "testimage"\n' features += 'TEST_SUITES = "ping ssh selftest"\n' + # We don't yet know what the server ip and port will be - they will be patched + # in at the start of the on-image test + features += 'PACKAGE_FEED_URIS = "http://bogus_ip:bogus_port"\n' + features += 'EXTRA_IMAGE_FEATURES += "package-management"\n' + features += 'PACKAGE_CLASSES = "package_rpm"' self.write_config(features) # Build core-image-sato and testimage
Signature · 1 line
2.11.0
PATCH 2/4 lib/oe/package_manager: import rpm signing key to rpmdb
From: Markus Lehtonen <[email protected]>
Import the gpg key used in rpm signing into rpmdb. This makes it possible again to create images when rpm signing is enabled.
Also, instruct dnf to enforce signature check if rpm signing is enabled.
--- meta/lib/oe/package_manager.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
meta/lib/oe/package_manager.py +10 -2
@@ -520,7 +520,14 @@ class RpmPM(PackageManager): open(platformconfdir + "macros", 'a').write("%_prefer_color 7") if self.d.getVar('RPM_SIGN_PACKAGES') == '1': - raise NotImplementedError("Signature verification with rpm not yet supported.") + pubkey_path = self.d.getVar('RPM_GPG_PUBKEY') + rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys") + cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + bb.fatal("Importing GPG key failed. Command '%s' " + "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) def create_configs(self): self._configure_dnf()
@@ -570,7 +577,8 @@ class RpmPM(PackageManager): output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) + (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) + (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) + - ["--nogpgcheck", "install"] + + (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) + + ["install"] + pkgs) failed_scriptlets_pkgnames = collections.OrderedDict()
Signature · 1 line
2.11.0
PATCH 3/4 selftest/signing.py: verify that images can be created from signed packages
In reply to Alexander Kanavin--- meta/lib/oeqa/selftest/signing.py | 8 ++++++++ 1 file changed, 8 insertions(+)
meta/lib/oeqa/selftest/signing.py +8 -0
@@ -34,8 +34,10 @@ class Signing(oeSelfTest): """ Summary: Test that packages can be signed in the package feed Expected: Package should be signed with the correct key + Expected: Images can be created from signed packages Product: oe-core Author: Daniel Istrate <[email protected]> + Author: Alexander Kanavin <[email protected]> AutomatedBy: Daniel Istrate <[email protected]> """ import oe.packagedata
@@ -85,6 +87,12 @@ class Signing(oeSelfTest): self.assertIn('rsa sha1 (md5) pgp md5 OK', ret.output, 'Package signed incorrectly.') shutil.rmtree(rpmdb) + #Check that an image can be built from signed packages + self.add_command_to_tearDown('bitbake -c clean core-image-minimal') + bitbake('-c clean core-image-minimal') + bitbake('core-image-minimal') + + @testcase(1382) def test_signing_sstate_archive(self): """
Signature · 1 line
2.11.0
PATCH 4/4 dnf: add /usr/bin/dnf symlink that points to /usr/bin/dnf-2
In reply to Alexander KanavinAll documentation refers to dnf binary as 'dnf' yet make install does not create one - it's done by Fedora's spec file when building the rpm. Let's replicate this behavior.
--- meta/lib/oe/package_manager.py | 2 +- meta/lib/oeqa/runtime/cases/dnf.py | 2 +- meta/recipes-devtools/dnf/dnf_git.bb | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-)
meta/lib/oe/package_manager.py +1 -1
@@ -677,7 +677,7 @@ class RpmPM(PackageManager): def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ): os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs - dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf-2") + dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf") standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")),
meta/lib/oeqa/runtime/cases/dnf.py +1 -1
@@ -12,7 +12,7 @@ from oeqa.runtime.decorator.package import OEHasPackage class DnfTest(OERuntimeTestCase): def dnf(self, command, expected = 0): - command = 'dnf-2 %s' % command + command = 'dnf %s' % command status, output = self.target.run(command, 1500) message = os.linesep.join([command, output]) self.assertEqual(status, expected, message)
meta/recipes-devtools/dnf/dnf_git.bb +9 -1
@@ -30,9 +30,17 @@ EXTRA_OECMAKE = " -DWITH_MAN=0 -DPYTHON_INSTALL_DIR=${PYTHON_SITEPACKAGES_DIR}" BBCLASSEXTEND = "native nativesdk" RDEPENDS_${PN}_class-target += "python-core python-codecs python-netclient python-email python-threading python-distutils librepo python-shell python-subprocess libcomps libdnf python-sqlite3 python-compression python-pygpgme python-backports-lzma python-rpm python-iniparse python-json python-importlib python-curses python-argparse" +# Create a symlink called 'dnf' as 'make install' does not do it, but +# .spec file in dnf source tree does (and then Fedora and dnf documentation +# says that dnf binary is plain 'dnf'). +do_install_append() { + ln -s -r ${D}/${bindir}/dnf-2 ${D}/${bindir}/dnf + ln -s -r ${D}/${bindir}/dnf-automatic-2 ${D}/${bindir}/dnf-automatic +} + # Direct dnf-native to read rpm configuration from our sysroot, not the one it was compiled in do_install_append_class-native() { - create_wrapper ${D}/${bindir}/dnf-2 \ + create_wrapper ${D}/${bindir}/dnf \ RPM_CONFIGDIR=${STAGING_LIBDIR_NATIVE}/rpm \ RPM_NO_CHROOT_FOR_SCRIPTS=1 }
Signature · 1 line
2.11.0