Skip to contentSkip to conversation

Fix distro_check when a distro list cannot be fetched

PATCH New Tim Orling 1 participant 4 messages Nov 14

lore
Share & tools
lore

Tim Orling

author
#0 lore raw

cover Fix distro_check when a distro list cannot be fetched

create_distro_packages_list() in meta/lib/oe/distro_check.py writes no package lists at all at the moment: the Debian list always fails, and any failed distro then crashes the loop with UnboundLocalError.

1: Skip a distro whose package list can't be fetched.
2: ftp.debian.org no longer has dists/stable-proposed-updates. Read
Debian updates from stable-updates instead, which is only published as Sources.xz.
3: Offline oe-selftest tests (oelib.distro_check) for both.

Nothing in OE-Core has used oe.distro_check since distrodata.bbclass was removed in 2018, but the Recipe Reporting System on the layer index still does (rrs_distros.py), to show which distros package each recipe, and it has been broken by this.

[YOCTO #16503]

Tim Orling (3):
lib/oe/distro_check: skip a distro whose package list can't be fetched lib/oe/distro_check: fetch Debian updates from stable-updates oeqa/selftest/oelib: add tests for oe.distro_check

 meta/lib/oe/distro_check.py                   |  9 ++-
 .../oeqa/selftest/cases/oelib/distro_check.py | 59 +++++++++++++++++++
 2 files changed, 65 insertions(+), 3 deletions(-)
 create mode 100644 meta/lib/oeqa/selftest/cases/oelib/distro_check.py
Signature · 1 line
2.56.1
author
#1 lore raw

PATCH 1/3 lib/oe/distro_check: skip a distro whose package list can't be fetched

When a fetcher raised, create_distro_packages_list() only warned and then used release and package_list anyway, failing with UnboundLocalError on the first distro, so no package lists were written at all. Move on to the next distro instead.

Tested by making the Debian fetcher raise: the other four package lists are written.

[YOCTO #16503]

AI-Generated: Claude Opus 5.5

---
 meta/lib/oe/distro_check.py | 1 +
 1 file changed, 1 insertion(+)
meta/lib/oe/distro_check.py +1 -0
index 1d78348ff3..56f81e630d 100644
@@ -170,6 +170,7 @@ def create_distro_packages_list(distro_check_dir, d):
             release, package_list = fetcher_func(d)
         except Exception as e:
             bb.warn("Cannot fetch packages for %s: %s" % (name, e))
+            continue
         bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
         if len(package_list) == 0:
             bb.error("Didn't fetch any packages for %s %s" % (name, release))
Signature · 1 line
2.56.1
author
#2 lore raw

PATCH 2/3 lib/oe/distro_check: fetch Debian updates from stable-updates

In reply to Tim Orling

ftp.debian.org no longer has dists/stable-proposed-updates, so the Debian package list always failed with HTTP 404. Use stable-updates, the equivalent of Ubuntu's -updates, which is only published as Sources.xz, so read .xz files with lzma.

Tested against ftp.debian.org: the Debian list is fetched again, with both main and updates packages.

[YOCTO #16503]

AI-Generated: Claude Opus 5.5

---
 meta/lib/oe/distro_check.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
meta/lib/oe/distro_check.py +5 -3
index 56f81e630d..223bed2ec2 100644
@@ -110,11 +110,13 @@ def find_latest_debian_release(url, d):
         return "_NotFound_"
 
 def get_debian_style_source_package_list(url, section, d):
-    "Return the list of package-names stored in the debian style Sources.gz file"
+    "Return the list of package-names stored in the debian style Sources.gz or Sources.xz file"
     import gzip
+    import lzma
 
+    opener = lzma.open if url.endswith(".xz") else gzip.open
     package_names = set()
-    for line in gzip.open(create_socket(url, d), mode="rt"):
+    for line in opener(create_socket(url, d), mode="rt"):
         if line.startswith("Package:"):
             pkg = line.split(":", 1)[1].strip()
             package_names.add(pkg + ":" + section)
@@ -125,7 +127,7 @@ def get_latest_released_debian_source_package_list(d):
     latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
     url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
     package_names = get_debian_style_source_package_list(url, "main", d)
-    url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
+    url = "http://ftp.debian.org/debian/dists/stable-updates/main/source/Sources.xz"
     package_names |= get_debian_style_source_package_list(url, "updates", d)
     return latest, package_names
 
Signature · 1 line
2.56.1
author
#3 lore raw

PATCH 3/3 oeqa/selftest/oelib: add tests for oe.distro_check

In reply to Tim Orling

Nothing in OE-Core uses oe.distro_check any more (its user is the Recipe Reporting System on the layer index), so nothing exercised it. Test, without network access, that create_distro_packages_list() skips a distro whose fetch fails and that Debian style Sources lists are read from both .gz and .xz files.

Without the two previous changes the skip and .xz tests fail.

[YOCTO #16503]

AI-Generated: Claude Opus 5.5

---
 .../oeqa/selftest/cases/oelib/distro_check.py | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 meta/lib/oeqa/selftest/cases/oelib/distro_check.py
meta/lib/oeqa/selftest/cases/oelib/distro_check.py new file +59 -0
new file mode 100644
index 0000000000..f47d1dd4a4
@@ -0,0 +1,59 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+import gzip
+import io
+import lzma
+import os
+import tempfile
+from unittest import mock
+from unittest.case import TestCase
+
+import bb
+# oe.distro_check uses bb as a builtin, which bb.build sets up
+import bb.build
+import oe.distro_check
+
+SOURCES = b"Package: foo\nVersion: 1.0\n\nPackage: bar\nVersion: 2.0\n"
+
+class TestDebianStyleSourcePackageList(TestCase):
+    def check(self, url, data):
+        with mock.patch.object(oe.distro_check, "create_socket", return_value=io.BytesIO(data)):
+            names = oe.distro_check.get_debian_style_source_package_list(url, "main", None)
+        self.assertEqual(names, {"foo:main", "bar:main"})
+
+    def test_gzip(self):
+        self.check("http://example.com/Sources.gz", gzip.compress(SOURCES))
+
+    def test_xz(self):
+        self.check("http://example.com/Sources.xz", lzma.compress(SOURCES))
+
+class TestCreateDistroPackagesList(TestCase):
+    def test_skip_failed_distro(self):
+        """
+        A distro whose package list can't be fetched is skipped, and the
+        other distros' lists are still written.
+        """
+        def fail(d):
+            raise OSError("HTTP Error 404: Not Found")
+
+        def ok(d):
+            return "1.0", {"foo:main"}
+
+        fetchers = {"get_latest_released_%s_source_package_list" % name: ok
+                    for name in ("ubuntu", "fedora", "opensuse", "clear")}
+        fetchers["get_latest_released_debian_source_package_list"] = fail
+
+        with tempfile.TemporaryDirectory() as tmpdir, \
+                mock.patch.multiple(oe.distro_check, **fetchers), \
+                mock.patch.object(bb, "note"), \
+                mock.patch.object(bb, "warn") as warn:
+            oe.distro_check.create_distro_packages_list(tmpdir, None)
+            lists = sorted(os.listdir(os.path.join(tmpdir, "package_lists")))
+
+        self.assertEqual(lists, ["Clear-1.0", "Fedora-1.0", "Ubuntu-1.0", "openSUSE-1.0"])
+        warn.assert_called_once()
+        self.assertIn("Debian", warn.call_args[0][0])
Signature · 1 line
2.56.1