distro_check: fix the Ubuntu and openSUSE package lists, drop Clear
cover distro_check: fix the Ubuntu and openSUSE package lists, drop Clear
With the fixes for bug 16503, create_distro_packages_list() still got the wrong Ubuntu release and no openSUSE or Clear lists:
NOTE: Distro: Ubuntu, Latest Release: xenial, # src packages: 3456
WARNING: Cannot fetch packages for openSUSE: HTTP Error 404: Not Found
WARNING: Cannot fetch packages for Clear: <urlopen error [Errno -3] Temporary failure in name resolution>
1: Bug 16509: the Ubuntu archive ignores the "newest first" sort that
find_latest_ubuntu_release() relied on, so it always took xenial. Use the release list that do-release-upgrade reads instead.
2: Bug 16510: download.opensuse.org's listings now link to "./15.6/",
still include Leap 42.x, and keep the source RPMs under repo/oss/src/; Leap 16 has no update repository.
3: Bug 16511: Clear Linux has been discontinued.
With these, all four lists are fetched:
NOTE: Distro: Debian, Latest Release: 13.7, # src packages: 37593
NOTE: Distro: Ubuntu, Latest Release: resolute, # src packages: 3053
NOTE: Distro: Fedora, Latest Release: 45, # src packages: 23237
NOTE: Distro: openSUSE, Latest Release: 16.1, # src packages: 10638
The new cases are covered by oe-selftest's oelib.distro_check, run with plain unittest (not oe-selftest).
This applies on top of "Fix distro_check when a distro list cannot be fetched" (bug 16503):
https://lore.kernel.org/all/[email protected]/
Tim Orling (3):
lib/oe/distro_check: find the latest Ubuntu release from meta-release lib/oe/distro_check: fix the openSUSE Leap package lists lib/oe/distro_check: drop Clear Linux
meta/lib/oe/distro_check.py | 62 ++++++++++++------- .../oeqa/selftest/cases/oelib/distro_check.py | 49 ++++++++++++++- 2 files changed, 87 insertions(+), 24 deletions(-)
Signature · 1 line
2.56.1
PATCH 1/3 lib/oe/distro_check: find the latest Ubuntu release from meta-release
find_latest_ubuntu_release() asked the archive for its dists/ listing newest first and took the first *-updates entry, but the archive ignores the sort and lists the oldest first, so Ubuntu was always xenial. Read the release list that do-release-upgrade uses instead, and take the last supported release.
Tested against changelogs.ubuntu.com and archive.ubuntu.com: resolute, with main and updates packages.
[YOCTO #16509]
AI-Generated: Claude Opus 5.5
--- meta/lib/oe/distro_check.py | 26 ++++++++++--------- .../oeqa/selftest/cases/oelib/distro_check.py | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-)
meta/lib/oe/distro_check.py +14 -12
@@ -131,23 +131,25 @@ def get_latest_released_debian_source_package_list(d): package_names |= get_debian_style_source_package_list(url, "updates", d) return latest, package_names -def find_latest_ubuntu_release(url, d): +def find_latest_ubuntu_release(d): """ - Find the latest listed Ubuntu release on the given ubuntu/dists/ URL. - - To avoid matching development releases look for distributions that have - updates, so the resulting distro could be any supported release. + Find the latest supported Ubuntu release, from the list of releases + that do-release-upgrade reads. The releases are listed oldest first. """ - url += "?C=M;O=D" # Descending Sort by Last Modified - for link in get_links_from_url(url, d): - if "-updates" in link: - distro = link.replace("-updates", "") - return distro - return "_NotFound_" + latest = "_NotFound_" + dist = None + listing = create_socket("https://changelogs.ubuntu.com/meta-release", d).read().decode("utf-8") + for line in listing.splitlines(): + key, _, value = line.partition(":") + if key == "Dist": + dist = value.strip() + elif key == "Supported" and value.strip() == "1" and dist: + latest = dist + return latest def get_latest_released_ubuntu_source_package_list(d): "Returns list of all the name os packages in the latest ubuntu distro" - latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d) + latest = find_latest_ubuntu_release(d) url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest package_names = get_debian_style_source_package_list(url, "main", d) url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
meta/lib/oeqa/selftest/cases/oelib/distro_check.py +26 -0
@@ -31,6 +31,32 @@ class TestDebianStyleSourcePackageList(TestCase): def test_xz(self): self.check("http://example.com/Sources.xz", lzma.compress(SOURCES)) +META_RELEASE = b"""Dist: noble +Name: Noble Numbat +Version: 24.04.5 LTS +Supported: 1 + +Dist: questing +Name: Questing Quokka +Version: 25.10 +Supported: 0 + +Dist: resolute +Name: Resolute Raccoon +Version: 26.04.1 LTS +Supported: 1 + +Dist: zzz +Name: Not supported any more +Version: 99.10 +Supported: 0 +""" + +class TestFindLatestUbuntuRelease(TestCase): + def test_latest_supported(self): + with mock.patch.object(oe.distro_check, "create_socket", return_value=io.BytesIO(META_RELEASE)): + self.assertEqual(oe.distro_check.find_latest_ubuntu_release(None), "resolute") + class TestCreateDistroPackagesList(TestCase): def test_skip_failed_distro(self): """
Signature · 1 line
2.56.1
PATCH 2/3 lib/oe/distro_check: fix the openSUSE Leap package lists
In reply to Tim Orlingdownload.opensuse.org's listings now link to "./15.6/", which find_latest_numeric_release() couldn't parse, so the Leap URLs had an empty release and failed with HTTP 404. Accept "./" links, ignore Leap 42.x (which came before 15.0), read the source RPMs from repo/oss/src/, where they now are, and allow for there being no update repository, as for Leap 16.
Tested against download.opensuse.org: Leap 16.1 is fetched again.
[YOCTO #16510]
AI-Generated: Claude Opus 5.5
--- meta/lib/oe/distro_check.py | 30 ++++++++++++++++--- .../oeqa/selftest/cases/oelib/distro_check.py | 19 ++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-)
meta/lib/oe/distro_check.py +26 -4
@@ -22,7 +22,11 @@ def get_links_from_url(url, d): soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a")) hyperlinks = [] for line in soup.find_all('a', href=True): - hyperlinks.append(line['href'].strip('/')) + href = line['href'] + # Some listings (e.g. MirrorCache) link to "./name/" + if href.startswith('./'): + href = href[2:] + hyperlinks.append(href.strip('/')) return hyperlinks def find_latest_numeric_release(url, d):
@@ -84,12 +88,30 @@ def get_latest_released_fedora_source_package_list(d): package_names |= get_source_package_list_from_url_by_letter("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d) return latest, package_names +def find_latest_opensuse_leap_release(url, d): + "Find the latest Leap release on the given url, ignoring Leap 42.x, which came before 15.0" + releases = [] + for link in get_links_from_url(url, d): + try: + version = tuple(int(n) for n in link.split(".")) + except ValueError: + continue + if version[0] != 42: + releases.append((version, link)) + return max(releases)[1] if releases else "_NotFound_" + def get_latest_released_opensuse_source_package_list(d): "Returns list of all the name os packages in the latest opensuse distro" - latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/leap", d) + from urllib.error import HTTPError + latest = find_latest_opensuse_leap_release("http://download.opensuse.org/source/distribution/leap/", d) - package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/suse/src/" % latest, "main", d) - package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) + package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/leap/%s/repo/oss/src/" % latest, "main", d) + # Leap 16 has no separate update repository + try: + package_names |= get_source_package_list_from_url("http://download.opensuse.org/update/leap/%s/oss/src/" % latest, "updates", d) + except HTTPError as e: + if e.code != 404: + raise return latest, package_names def get_latest_released_clear_source_package_list(d):
meta/lib/oeqa/selftest/cases/oelib/distro_check.py +19 -0
@@ -57,6 +57,25 @@ class TestFindLatestUbuntuRelease(TestCase): with mock.patch.object(oe.distro_check, "create_socket", return_value=io.BytesIO(META_RELEASE)): self.assertEqual(oe.distro_check.find_latest_ubuntu_release(None), "resolute") +class TestOpenSUSE(TestCase): + LISTING = b"""<a href="../">..</a> <a href="./15.0-Current/">15.0-Current/</a> +<a href="./15.6/">15.6/</a> <a href="./16.0/">16.0/</a> <a href="./42.3/">42.3/</a>""" + + def test_links_relative_to_current_directory(self): + try: + from bs4 import BeautifulSoup, SoupStrainer + except ImportError: + self.skipTest("bs4 is not available") + with mock.patch.object(oe.distro_check, "create_socket", return_value=io.BytesIO(self.LISTING)): + links = oe.distro_check.get_links_from_url("http://example.com/leap/", None) + self.assertEqual(links, ["..", "15.0-Current", "15.6", "16.0", "42.3"]) + + def test_latest_leap_release(self): + # Leap 42.x came before 15.0 + links = ["..", "15.0-Current", "15.6", "16.0", "16.1", "42.2", "42.3", "42.3-Current"] + with mock.patch.object(oe.distro_check, "get_links_from_url", return_value=links): + self.assertEqual(oe.distro_check.find_latest_opensuse_leap_release("http://example.com/leap/", None), "16.1") + class TestCreateDistroPackagesList(TestCase): def test_skip_failed_distro(self): """
Signature · 1 line
2.56.1
Re: [OE-core] [PATCH 2/3] lib/oe/distro_check: fix the openSUSE Leap package lists
Quoted 24 lines · Tim Orling via openembedded-core
download.opensuse.org's listings now link to "./15.6/", which
find_latest_numeric_release()couldn't parse, so the Leap URLs had an empty release and failed with HTTP 404. Accept "./" links, ignore Leap 42.x (which came before 15.0), read the source RPMs from repo/oss/src/, where they now are, and allow for there being no update repository, as for Leap 16.Tested against download.opensuse.org: Leap 16.1 is fetched again.
[YOCTO #16510]
AI-Generated: Claude Opus 5.5
Signed-off-by Tim Orling <[email protected]>---
meta/lib/oe/distro_check.py | 30 ++++++++++++++++---.../oeqa/selftest/cases/oelib/distro_check.py| 19 ++++++++++++2 files changed, 45 insertions(+), 4 deletions(-)meta/lib/oe/distro_check.py +0
-0@@ -22,7 +22,11 @@ def get_links_from_url(url, d): soup = BeautifulSoup(create_socket(url,d), "html.parser", parse_only=SoupStrainer("a"))
Quoted 7 lines
hyperlinks = []
for line insoup.find_all('a', href=True):
- hyperlinks.append(line['href'].strip('/')) + href = line['href']
+ # Some listings (e.g. MirrorCache) link to "./name/" + ifhref.startswith('./'):
+ href = href[2:]
Being slightly picky, we could use href.removeprefix('./') here which is slightly neater?
I mention it mainly as I think we can get better about using that and removesuffix() and the code is better for it...
Quoted 2 lines
+ hyperlinks.append(href.strip('/'))
return hyperlinks
Cheers,
Richard
PATCH 3/3 lib/oe/distro_check: drop Clear Linux
In reply to Tim OrlingClear Linux OS was discontinued in 2025 and download.clearlinux.org no longer resolves, so its package list could only ever fail.
[YOCTO #16511]
AI-Generated: Claude Opus 5.5
--- meta/lib/oe/distro_check.py | 6 ------ meta/lib/oeqa/selftest/cases/oelib/distro_check.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-)
meta/lib/oe/distro_check.py +0 -6
@@ -114,11 +114,6 @@ def get_latest_released_opensuse_source_package_list(d): raise return latest, package_names -def get_latest_released_clear_source_package_list(d): - latest = find_latest_numeric_release("https://download.clearlinux.org/releases/", d) - package_names = get_source_package_list_from_url("https://download.clearlinux.org/releases/%s/clear/source/SRPMS/" % latest, "main", d) - return latest, package_names - def find_latest_debian_release(url, d): "Find the latest listed debian release on the given url"
@@ -188,7 +183,6 @@ def create_distro_packages_list(distro_check_dir, d): ("Ubuntu", get_latest_released_ubuntu_source_package_list), ("Fedora", get_latest_released_fedora_source_package_list), ("openSUSE", get_latest_released_opensuse_source_package_list), - ("Clear", get_latest_released_clear_source_package_list), ) for name, fetcher_func in per_distro_functions:
meta/lib/oeqa/selftest/cases/oelib/distro_check.py +2 -2
@@ -89,7 +89,7 @@ class TestCreateDistroPackagesList(TestCase): return "1.0", {"foo:main"} fetchers = {"get_latest_released_%s_source_package_list" % name: ok - for name in ("ubuntu", "fedora", "opensuse", "clear")} + for name in ("ubuntu", "fedora", "opensuse")} fetchers["get_latest_released_debian_source_package_list"] = fail with tempfile.TemporaryDirectory() as tmpdir, \
@@ -99,6 +99,6 @@ class TestCreateDistroPackagesList(TestCase): oe.distro_check.create_distro_packages_list(tmpdir, None) lists = sorted(os.listdir(os.path.join(tmpdir, "package_lists"))) - self.assertEqual(lists, ["Clear-1.0", "Fedora-1.0", "Ubuntu-1.0", "openSUSE-1.0"]) + self.assertEqual(lists, ["Fedora-1.0", "Ubuntu-1.0", "openSUSE-1.0"]) warn.assert_called_once() self.assertIn("Debian", warn.call_args[0][0])
Signature · 1 line
2.56.1