oeqa/utils/gitarchive: put back local tags listing as third tag listing method
cover oeqa/utils/gitarchive: put back local tags listing as third tag listing method
Hello,
this is another attempt to properly fix tag computation in Autobuilder.
First attempt ([1]) completely replaced "git tag" with "git ls-remote" to make sure to know about all tags when computing new tag name. It raised some issue because oe-git-archive works with local repositories with no remote configured (it uses explicit remote url in git calls), so a second series introduced this provided url usage as a fallback ([2])
There are still scenarios which do not match the two above: [3] shows that oe-build-perf-report works with directories with no remote configured (the same directories as oe-git-archive, in fact), AND does not use urls (I guess it uses accumulated result from previous run, but I got no access to the AB machines to make sure about it). I feel that it may not be relevant to try to force-propagate fallback urls to oe-build-perf-report, moreover there may be other unidentified cases scattered in whole AB configuration which would need the same fix. Hence this series proposes to reintroduce the previous method to retrieve tags ("git tag") as last resort method, with a warning when reached, to both fix current issues and allow to diagnostic future issues if it makes the initial tagging issue appear again.
[1] https://lore.kernel.org/openembedded-core/[email protected]/
[2] https://lore.kernel.org/openembedded-core/[email protected]/
[3] https://autobuilder.yoctoproject.org/typhoon/#/builders/133/builds/2125/steps/12/logs/stdio
Alexis Lothoré (2):
oeqa/utils/gitarchive: allow to pass a logger to get_tags oeqa/utils/gitarchive: fall back to local tags when listing existing
tags
.../oeqa/selftest/cases/gitarchivetests.py | 25 +++++++++----- meta/lib/oeqa/utils/gitarchive.py | 34 ++++++++++++------- 2 files changed, 38 insertions(+), 21 deletions(-)
Signature · 1 line
2.41.0
PATCH 1/2 oeqa/utils/gitarchive: allow to pass a logger to get_tags
From: Alexis Lothoré <[email protected]>
Propagate a "log" parameter to get_tags in order to know what method is used to retrieve existing tags
--- meta/lib/oeqa/selftest/cases/gitarchivetests.py | 13 ++++++++++--- meta/lib/oeqa/utils/gitarchive.py | 11 ++++++----- 2 files changed, 16 insertions(+), 8 deletions(-)
meta/lib/oeqa/selftest/cases/gitarchivetests.py +10 -3
@@ -14,6 +14,7 @@ from oeqa.utils.git import GitError import tempfile import shutil import scriptutils +import logging from oeqa.selftest.case import OESelftestTestCase logger = scriptutils.logger_create('resulttool')
@@ -58,6 +59,12 @@ class GitArchiveTests(OESelftestTestCase): TEST_COMMIT="0f7d5df" TEST_COMMIT_COUNT="42" + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.log = logging.getLogger('gitarchivetests') + cls.log.setLevel(logging.DEBUG) + def test_create_first_test_tag(self): path, git_obj = create_fake_repository(False) keywords = {'commit': self.TEST_COMMIT, 'branch': self.TEST_BRANCH, "commit_count": self.TEST_COMMIT_COUNT}
@@ -101,7 +108,7 @@ class GitArchiveTests(OESelftestTestCase): url = 'git://git.yoctoproject.org/poky' path, git_obj = create_fake_repository(False, None, False) - tags = ga.get_tags(git_obj, pattern="yocto-*", url=url) + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) """Test for some well established tags (released tags)""" self.assertIn("yocto-4.0", tags) self.assertIn("yocto-4.1", tags)
@@ -114,7 +121,7 @@ class GitArchiveTests(OESelftestTestCase): """Test for some well established tags (released tags)""" with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, pattern="yocto-*") + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*") delete_fake_repository(path) def test_get_tags_without_valid_remote_and_wrong_url(self):
@@ -123,5 +130,5 @@ class GitArchiveTests(OESelftestTestCase): """Test for some well established tags (released tags)""" with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, pattern="yocto-*", url=url) + tags = ga.get_tags(git_obj, self.log, pattern="yocto-*", url=url) delete_fake_repository(path)
meta/lib/oeqa/utils/gitarchive.py +6 -5
@@ -100,7 +100,7 @@ def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): if os.path.exists(tmp_index): os.unlink(tmp_index) -def get_tags(repo, pattern=None, url=None): +def get_tags(repo, log, pattern=None, url=None): """ Fetch remote tags from current repository A pattern can be provided to filter returned tags list
@@ -120,6 +120,7 @@ def get_tags(repo, pattern=None, url=None): # If it fails, retry with repository url if one is provided if not url: raise(e) + log.info("No remote repository configured, use provided url") cmd = base_cmd.copy() cmd.append(url) if pattern:
@@ -129,7 +130,7 @@ def get_tags(repo, pattern=None, url=None): return ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, - url, keywords): + url, log, keywords): """Generate tag name and message, with support for running id number""" keyws = keywords.copy() # Tag number is handled specially: if not defined, we autoincrement it
@@ -143,7 +144,7 @@ def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') keyws['tag_number'] = 0 - for existing_tag in get_tags(repo, url=url): + for existing_tag in get_tags(repo, log, url=url): match = re.match(tag_re, existing_tag) if match and int(match.group('tag_number')) >= keyws['tag_number']:
@@ -171,7 +172,7 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms tag_name, tag_msg = expand_tag_strings(data_repo, tagname, tag_msg_subject, tag_msg_body, - push, keywords) + push, log, keywords) # Commit data commit = git_commit_data(data_repo, data_dir, branch_name,
@@ -209,7 +210,7 @@ def get_test_runs(log, repo, tag_name, **kwargs): # Get a list of all matching tags tag_pattern = tag_name.format(**str_fields) - tags = get_tags(repo, pattern=tag_pattern) + tags = get_tags(repo, log, pattern=tag_pattern) log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern) # Parse undefined fields from tag names
Signature · 1 line
2.41.0
PATCH 2/2 oeqa/utils/gitarchive: fall back to local tags when listing existing tags
In reply to Alexis LothoréFrom: Alexis Lothoré <[email protected]>
e9cff55e73cc has switched tag listing from bare "git tag" to "git ls-remote" to make sure not to miss remote tags which are not fetched locally. This mechanism first checks for configured remote repository, next for possibly passed url, and then fails if none worked.
However there are still cases where no remote repository is configured and no url is provided (for instance: buildperf tests use an empty git directory to store tests). Fix those cases by putting back the old behavior (local tags check) as last resort, with at least a warning for future diagnostics if we still encounter tagging issues
--- .../oeqa/selftest/cases/gitarchivetests.py | 14 ++++++----- meta/lib/oeqa/utils/gitarchive.py | 25 ++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-)
meta/lib/oeqa/selftest/cases/gitarchivetests.py +8 -6
@@ -115,13 +115,15 @@ class GitArchiveTests(OESelftestTestCase): self.assertIn("yocto-4.2", tags) delete_fake_repository(path) - def test_get_tags_without_valid_remote_neither_url(self): - url = 'git://git.yoctoproject.org/poky' - path, git_obj = create_fake_repository(False, None, False) + def test_get_tags_with_only_local_tag(self): + fake_tags_list=["main/10-g0f7d5df/0", "main/10-g0f7d5df/1", "foo/20-g2468f5d/0"] + path, git_obj = create_fake_repository(True, fake_tags_list, False) - """Test for some well established tags (released tags)""" - with self.assertRaises(GitError): - tags = ga.get_tags(git_obj, self.log, pattern="yocto-*") + """No remote is configured and no url is passed: get_tags must fall + back to local tags + """ + tags = ga.get_tags(git_obj, self.log) + self.assertCountEqual(tags, fake_tags_list) delete_fake_repository(path) def test_get_tags_without_valid_remote_and_wrong_url(self):
meta/lib/oeqa/utils/gitarchive.py +16 -9
@@ -116,18 +116,25 @@ def get_tags(repo, log, pattern=None, url=None): cmd.append(pattern) try: tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] except GitError as e: # If it fails, retry with repository url if one is provided - if not url: - raise(e) - log.info("No remote repository configured, use provided url") - cmd = base_cmd.copy() - cmd.append(url) - if pattern: - cmd.append(pattern) - tags_refs = repo.run_cmd(cmd) + if url: + log.info("No remote repository configured, use provided url") + cmd = base_cmd.copy() + cmd.append(url) + if pattern: + cmd.append(pattern) + tags_refs = repo.run_cmd(cmd) + tags = ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + else: + log.warning("Read local tags only, some remote tags may be missed") + cmd = ["tag"] + if pattern: + cmd += ["-l", pattern] + tags = repo.run_cmd(cmd).splitlines() - return ["".join(d.split()[1].split('/', 2)[2:]) for d in tags_refs.splitlines()] + return tags def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, url, log, keywords):
Signature · 1 line
2.41.0