Skip to contentSkip to conversation

rpm: add support for remote package feeds via PACKAGE_FEED_URIS variable

PATCH New Alexander Kanavin 1 participant 4 messages Mar 16, 2017

lore
Share & tools
lore
#0 lore raw

PATCH 1/4 rpm: add support for remote package feeds via PACKAGE_FEED_URIS variable

I've used a previous patch (which was never merged) by Humberto Ibarra <[email protected]> as a model for how to do runtime testing of this feature (e.g. we need to boot an image, run dnf on it, and check that it is indeed able to access the remote repo over http). Here's his original commit message:

=====
Testing that feeds specified with PACKAGE_FEED_URIS var are set correctly has two parts. First a build with this var set is required, and then smart update needs to be issued in the running taget.

The previous is not a common selftest practice because this is a simple test, but requires building and running a specific image, which takes a lot of time. testimage is not a good fit either, since the images tested there do not have the PACKAGE_FEED_URIS var set.

For this test, the runtime-test module is being used, which is a selftest module but runs a testimage command. The var and test environment were set in runtime-perf.py and the actual test is done in a new testcase added to meta-selftest layer.
=====

[YOCTO #10872]

---
 meta-selftest/lib/oeqa/runtime/cases/selftest.py | 42 ++++++++++++++++++++++++
 meta/lib/oe/package_manager.py                   | 12 ++++++-
 meta/lib/oeqa/selftest/runtime-test.py           |  7 ++++
 3 files changed, 60 insertions(+), 1 deletion(-)
meta-selftest/lib/oeqa/runtime/cases/selftest.py +42 -0
index 19de7406236..e4985a6edd3 100644
@@ -1,5 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
+from oeqa.runtime.cases.dnf import DnfTest
+from oeqa.utils.httpserver import HTTPService
 
 class Selftest(OERuntimeTestCase):
 
@@ -29,3 +31,43 @@ class Selftest(OERuntimeTestCase):
 
         (status, output) = self.target.run("socat -V")
         self.assertNotEqual(status, 0, msg="socat is still installed")
+
+
+class DnfSelftest(DnfTest):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-rootfs-repo'),
+                                      cls.tc.target.server_ip)
+        cls.repo_server.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.repo_server.stop()
+
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    def test_verify_package_feeds(self):
+        """
+        Summary: Check correct setting of PACKAGE_FEED_URIS var
+        Expected: 1. Feeds were correctly set for dnf
+                  2. Update recovers packages from host's repo
+        Author: Humberto Ibarra <[email protected]>
+        Author: Alexander Kanavin <[email protected]>
+        """
+        # When we created an image, we had to supply fake ip and port
+        # for the feeds. Now we can patch the real ones into the config file.
+        import tempfile
+        temp_file = tempfile.TemporaryDirectory(prefix="oeqa-remotefeeds-").name
+        self.tc.target.copyFrom("/etc/yum.repos.d/oe-remote-repo.repo", temp_file)
+        fixed_config = open(temp_file, "r").read().replace("bogus_ip", self.tc.target.server_ip).replace("bogus_port", str(self.repo_server.port))
+        open(temp_file, "w").write(fixed_config)
+        self.tc.target.copyTo(temp_file, "/etc/yum.repos.d/oe-remote-repo.repo")
+
+        import re
+        output_makecache = self.dnf('makecache')
+        self.assertTrue(re.match(r".*Metadata cache created", output_makecache, re.DOTALL) is not None, msg = "dnf makecache failed: %s" %(output_makecache))
+
+        output_repoinfo = self.dnf('repoinfo')
+        matchobj = re.match(r".*Repo-pkgs\s*:\s*(?P<n_pkgs>[0-9]+)", output_repoinfo, re.DOTALL)
+        self.assertTrue(matchobj is not None, msg = "Could not find the amount of packages in dnf repoinfo output: %s" %(output_repoinfo))
+        self.assertTrue(int(matchobj.group('n_pkgs')) > 0, msg = "Amount of remote packages is not more than zero: %s\n" %(output_repoinfo))
meta/lib/oe/package_manager.py +11 -1
index d51609189d0..b016bc32dc5 100644
@@ -533,10 +533,20 @@ class RpmPM(PackageManager):
         bb.utils.unlockfile(lf)
 
     def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
+        from urllib.parse import urlparse
+
         if feed_uris == "":
             return
 
-        raise NotImplementedError("Adding remote dnf feeds not yet supported.")
+        bb.utils.mkdirhier(oe.path.join(self.target_rootfs, "etc", "yum.repos.d"))
+        remote_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
+        for uri in remote_uris:
+            repo_name = "oe-remote-repo" + "-".join(urlparse(uri).path.split("/"))
+            if feed_archs is not None:
+                repo_uris = [uri + "/" + arch for arch in feed_archs]
+            else:
+                repo_uris = [uri]
+            open(oe.path.join(self.target_rootfs, "etc", "yum.repos.d", repo_name + ".repo"), 'w').write("[%s]\nbaseurl=%s\n" % (repo_name, " ".join(repo_uris)))
 
     def _prepare_pkg_transaction(self):
         os.environ['D'] = self.target_rootfs
meta/lib/oeqa/selftest/runtime-test.py +7 -0
index e8b483d7f8b..171a37329f5 100644
@@ -108,14 +108,21 @@ class TestImage(oeSelfTest):
         Summary: Check install packages functionality for testimage/testexport.
         Expected: 1. Import tests from a directory other than meta.
                   2. Check install/uninstall of socat.
+                  3. Check that remote package feeds can be accessed
         Product: oe-core
         Author: Mariano Lopez <[email protected]>
+        Author: Alexander Kanavin <[email protected]>
         """
         if get_bb_var('DISTRO') == 'poky-tiny':
             self.skipTest('core-image-full-cmdline not buildable for poky-tiny')
 
         features = 'INHERIT += "testimage"\n'
         features += 'TEST_SUITES = "ping ssh selftest"\n'
+        # We don't yet know what the server ip and port will be - they will be patched
+        # in at the start of the on-image test
+        features += 'PACKAGE_FEED_URIS = "http://bogus_ip:bogus_port"\n'
+        features += 'EXTRA_IMAGE_FEATURES += "package-management"\n'
+        features += 'PACKAGE_CLASSES = "package_rpm"'
         self.write_config(features)
 
         # Build core-image-sato and testimage
Signature · 1 line
2.11.0
#1 lore raw

PATCH 2/4 lib/oe/package_manager: import rpm signing key to rpmdb

From: Markus Lehtonen <[email protected]>

Import the gpg key used in rpm signing into rpmdb. This makes it possible again to create images when rpm signing is enabled.

Also, instruct dnf to enforce signature check if rpm signing is enabled.

---
 meta/lib/oe/package_manager.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
meta/lib/oe/package_manager.py +10 -2
index b016bc32dc5..eeb4c76071a 100644
@@ -520,7 +520,14 @@ class RpmPM(PackageManager):
             open(platformconfdir + "macros", 'a').write("%_prefer_color 7")
 
         if self.d.getVar('RPM_SIGN_PACKAGES') == '1':
-            raise NotImplementedError("Signature verification with rpm not yet supported.")
+            pubkey_path = self.d.getVar('RPM_GPG_PUBKEY')
+            rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmkeys")
+            cmd = [rpm_bin, '--root=%s' % self.target_rootfs, '--import', pubkey_path]
+            try:
+                subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                bb.fatal("Importing GPG key failed. Command '%s' "
+                        "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
 
     def create_configs(self):
         self._configure_dnf()
@@ -570,7 +577,8 @@ class RpmPM(PackageManager):
         output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) +
                          (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) +
                          (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) +
-                         ["--nogpgcheck", "install"] +
+                         (["--nogpgcheck"] if self.d.getVar('RPM_SIGN_PACKAGES') != '1' else ["--setopt=gpgcheck=True"]) +
+                         ["install"] +
                          pkgs)
 
         failed_scriptlets_pkgnames = collections.OrderedDict()
Signature · 1 line
2.11.0
#2 lore raw

PATCH 3/4 selftest/signing.py: verify that images can be created from signed packages

In reply to Alexander Kanavin
---
 meta/lib/oeqa/selftest/signing.py | 8 ++++++++
 1 file changed, 8 insertions(+)
meta/lib/oeqa/selftest/signing.py +8 -0
index 38b2fca8cf9..8a106aa80f6 100644
@@ -34,8 +34,10 @@ class Signing(oeSelfTest):
         """
         Summary:     Test that packages can be signed in the package feed
         Expected:    Package should be signed with the correct key
+        Expected:    Images can be created from signed packages
         Product:     oe-core
         Author:      Daniel Istrate <[email protected]>
+        Author:      Alexander Kanavin <[email protected]>
         AutomatedBy: Daniel Istrate <[email protected]>
         """
         import oe.packagedata
@@ -85,6 +87,12 @@ class Signing(oeSelfTest):
         self.assertIn('rsa sha1 (md5) pgp md5 OK', ret.output, 'Package signed incorrectly.')
         shutil.rmtree(rpmdb)
 
+        #Check that an image can be built from signed packages
+        self.add_command_to_tearDown('bitbake -c clean core-image-minimal')
+        bitbake('-c clean core-image-minimal')
+        bitbake('core-image-minimal')
+
+
     @testcase(1382)
     def test_signing_sstate_archive(self):
         """
Signature · 1 line
2.11.0
#3 lore raw

PATCH 4/4 dnf: add /usr/bin/dnf symlink that points to /usr/bin/dnf-2

In reply to Alexander Kanavin

All documentation refers to dnf binary as 'dnf' yet make install does not create one - it's done by Fedora's spec file when building the rpm. Let's replicate this behavior.

---
 meta/lib/oe/package_manager.py       |  2 +-
 meta/lib/oeqa/runtime/cases/dnf.py   |  2 +-
 meta/recipes-devtools/dnf/dnf_git.bb | 10 +++++++++-
 3 files changed, 11 insertions(+), 3 deletions(-)
meta/lib/oe/package_manager.py +1 -1
index eeb4c76071a..ae6849bce3f 100644
@@ -677,7 +677,7 @@ class RpmPM(PackageManager):
     def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ):
         os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs
 
-        dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf-2")
+        dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf")
         standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y",
                              "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"),
                              "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")),
meta/lib/oeqa/runtime/cases/dnf.py +1 -1
index 77e7eb94507..59a263dc63a 100644
@@ -12,7 +12,7 @@ from oeqa.runtime.decorator.package import OEHasPackage
 class DnfTest(OERuntimeTestCase):
 
     def dnf(self, command, expected = 0):
-        command = 'dnf-2 %s' % command
+        command = 'dnf %s' % command
         status, output = self.target.run(command, 1500)
         message = os.linesep.join([command, output])
         self.assertEqual(status, expected, message)
meta/recipes-devtools/dnf/dnf_git.bb +9 -1
index 018352a6c9b..3a2cb635e40 100644
@@ -30,9 +30,17 @@ EXTRA_OECMAKE = " -DWITH_MAN=0 -DPYTHON_INSTALL_DIR=${PYTHON_SITEPACKAGES_DIR}"
 BBCLASSEXTEND = "native nativesdk"
 RDEPENDS_${PN}_class-target += "python-core python-codecs python-netclient python-email python-threading python-distutils librepo python-shell python-subprocess libcomps libdnf python-sqlite3 python-compression python-pygpgme python-backports-lzma python-rpm python-iniparse python-json python-importlib python-curses python-argparse"
 
+# Create a symlink called 'dnf' as 'make install' does not do it, but
+# .spec file in dnf source tree does (and then Fedora and dnf documentation
+# says that dnf binary is plain 'dnf').
+do_install_append() {
+        ln -s -r ${D}/${bindir}/dnf-2 ${D}/${bindir}/dnf
+        ln -s -r ${D}/${bindir}/dnf-automatic-2 ${D}/${bindir}/dnf-automatic
+}
+
 # Direct dnf-native to read rpm configuration from our sysroot, not the one it was compiled in
 do_install_append_class-native() {
-        create_wrapper ${D}/${bindir}/dnf-2 \
+        create_wrapper ${D}/${bindir}/dnf \
                 RPM_CONFIGDIR=${STAGING_LIBDIR_NATIVE}/rpm \
                 RPM_NO_CHROOT_FOR_SCRIPTS=1
 }
Signature · 1 line
2.11.0