Follow-up fixes for command argument conversions
cover Follow-up fixes for command argument conversions
While testing the recent work to replace shell command strings with argument lists I found a few places where behavior previously provided by the shell was not fully represented in argv form. This series contains small, localized corrections which is supposed to restore the previous command semantics.
Anders Heimer (9):
fetch2/az: fix wget command argument construction fetch2/git: avoid an empty argument to git lfs fetch fetch2/git: do not shell-quote the shallow clone remote fetch2/git: keep the gitpkgv commit count numeric fetch2/gitannex: split the annex get arguments fetch2/repo: fix mirror tar command arguments fetch2/sftp: fix port and remote command arguments fetch2/svn: remove shell quotes from the tar exclusion
toaster: preserve shell execution for build launch commands
lib/bb/fetch2/az.py | 10 +++++----- lib/bb/fetch2/git.py | 10 ++++++---- lib/bb/fetch2/gitannex.py | 2 +- lib/bb/fetch2/repo.py | 7 +++++-- lib/bb/fetch2/sftp.py | 4 ++-- lib/bb/fetch2/svn.py | 2 +- lib/toaster/bldcontrol/localhostbecontroller.py | 8 ++++---- 7 files changed, 24 insertions(+), 19 deletions(-)
PATCH 1/9 fetch2/az: fix wget command argument construction
--- lib/bb/fetch2/az.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
lib/bb/fetch2/az.py +5 -5
@@ -48,17 +48,17 @@ class Az(Wget): # If were reaching the account transaction limit we might be refused a connection, # retrying allows us to avoid false negatives since the limit changes over time - fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' + fetchcmd = self.basecmd + ['--retry-connrefused', '--waitretry=5'] # We need to provide a localpath to avoid wget using the SAS # ud.localfile either has the downloadfilename or ud.path localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) bb.utils.mkdirhier(os.path.dirname(localpath)) - fetchcmd += " -O %s" % shlex.quote(localpath) + fetchcmd += ["-O", localpath] if ud.user and ud.pswd: - fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) + fetchcmd += ["--user=%s" % ud.user, "--password=%s" % ud.pswd, "--auth-no-challenge"] # Check if a Shared Access Signature was given and use it az_sas = d.getVar('AZ_SAS')
@@ -73,9 +73,9 @@ class Az(Wget): dldir = d.getVar("DL_DIR") if os.path.exists(ud.localpath): # file exists, but we didnt complete it.. trying again. - fetchcmd += " -c -P %s '%s'" % (dldir, azuri) + fetchcmd += ["-c", "-P", dldir, azuri] else: - fetchcmd += " -P %s '%s'" % (dldir, azuri) + fetchcmd += ["-P", dldir, azuri] try: self._runwget(ud, d, fetchcmd, False)
PATCH 2/9 fetch2/git: avoid an empty argument to git lfs fetch
In reply to Anders Heimer--- lib/bb/fetch2/git.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/bb/fetch2/git.py +3 -1
@@ -505,7 +505,9 @@ class Git(FetchMethod): # Using worktree with the revision because .lfsconfig may exists worktree_add_cmd = ud.basecmd + ['worktree', 'add', 'wt', revision] runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir) - lfs_fetch_cmd = ud.basecmd + ['lfs', 'fetch', "--all" if fetchall else ""] + lfs_fetch_cmd = ud.basecmd + ['lfs', 'fetch'] + if fetchall: + lfs_fetch_cmd.append('--all') runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt")) worktree_rem_cmd = ud.basecmd + ['worktree', 'remove', '-f', 'wt'] runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
PATCH 3/9 fetch2/git: do not shell-quote the shallow clone remote
In reply to Anders Heimer--- lib/bb/fetch2/git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/git.py +1 -1
@@ -580,7 +580,7 @@ class Git(FetchMethod): # Use repourl when creating a fast initial shallow clone # Prefer already existing full bare clones if available if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir): - remote = shlex.quote(repourl) + remote = repourl else: remote = ud.clonedir runfetchcmd(ud.basecmd + ['remote', 'add', 'origin', remote], d, workdir=dest)
PATCH 4/9 fetch2/git: keep the gitpkgv commit count numeric
In reply to Anders HeimerThe cold-cache path used to keep the raw splitlines() list in commits while only using len(commits) when writing the cache file.
That same commits value is formatted into the returned version string below, so cold cache could return a Python list representation while warm cache returned the numeric count. Store the numeric count in commits before both writing the cache and building the return value.
--- lib/bb/fetch2/git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/bb/fetch2/git.py +2 -2
@@ -1031,9 +1031,9 @@ class Git(FetchMethod): commits = None else: if not os.path.exists(rev_file) or not os.path.getsize(rev_file): - commits = bb.fetch2.runfetchcmd(['git', 'rev-list', rev, '--'], d).splitlines() + commits = len(bb.fetch2.runfetchcmd(['git', 'rev-list', rev, '--'], d).splitlines()) if commits: - open(rev_file, "w").write("%d\n" % len(commits)) + open(rev_file, "w").write("%d\n" % commits) else: commits = open(rev_file, "r").readline(128).strip() if commits:
PATCH 5/9 fetch2/gitannex: split the annex get arguments
In reply to Anders Heimer--- lib/bb/fetch2/gitannex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/gitannex.py +1 -1
@@ -36,7 +36,7 @@ class GitANNEX(Git): def update_annex(self, ud, d, wd): try: - runfetchcmd(ud.basecmd + ['annex get', '--all'], d, quiet=True, workdir=wd) + runfetchcmd(ud.basecmd + ['annex', 'get', '--all'], d, quiet=True, workdir=wd) except bb.fetch.FetchError: return False runfetchcmd(['chmod', 'u+w', '-R', '%s/annex' % ud.clonedir], d, quiet=True, workdir=wd)
PATCH 6/9 fetch2/repo: fix mirror tar command arguments
In reply to Anders HeimerThe direct tar invocation no longer has a shell to expand ./* or strip quotes from exclude patterns. Build the path list explicitly and pass tar exclude patterns without shell quote characters.
--- lib/bb/fetch2/repo.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
lib/bb/fetch2/repo.py +5 -2
@@ -73,10 +73,13 @@ class Repo(FetchMethod): if scmdata == "keep": tar_flags = [] else: - tar_flags = ["--exclude='.repo'", "--exclude='.git'"] + tar_flags = ["--exclude=.repo", "--exclude=.git"] # Create a cache - runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath, os.path.join(".", "*")], d, workdir=codir) + tar_paths = [os.path.join(".", name) + for name in sorted(os.listdir(codir)) + if not name.startswith(".")] + runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath] + tar_paths, d, workdir=codir) def supports_srcrev(self): return False
PATCH 7/9 fetch2/sftp: fix port and remote command arguments
In reply to Anders Heimer--- lib/bb/fetch2/sftp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/bb/fetch2/sftp.py +2 -2
@@ -85,7 +85,7 @@ class SFTP(FetchMethod): urlo = URI(ud.url) basecmd = ['sftp', '-oBatchMode=yes'] if urlo.port: - basecmd += ['-P', urlo.port] + basecmd += ['-P', str(urlo.port)] urlo.port = None dldir = d.getVar('DL_DIR')
@@ -102,7 +102,7 @@ class SFTP(FetchMethod): if path[:3] == '/~/': path = path[3:] - remote = '"%s%s:%s"' % (user, urlo.hostname, path) + remote = '%s%s:%s' % (user, urlo.hostname, path) cmd = basecmd + [remote, lpath]
PATCH 8/9 fetch2/svn: remove shell quotes from the tar exclusion
In reply to Anders Heimer--- lib/bb/fetch2/svn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/svn.py +1 -1
@@ -170,7 +170,7 @@ class Svn(FetchMethod): if scmdata == "keep": tar_flags = [] else: - tar_flags = ["--exclude='.svn'"] + tar_flags = ["--exclude=.svn"] # tar them up to a defined filename runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath, ud.path_spec], d,
PATCH 9/9 toaster: preserve shell execution for build launch commands
In reply to Anders Heimer--- lib/toaster/bldcontrol/localhostbecontroller.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
lib/toaster/bldcontrol/localhostbecontroller.py +4 -4
@@ -547,14 +547,14 @@ class LocalhostBEController(BuildEnvironmentController): local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 'bitbake') if not is_merged_attr: - self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" ' + self._shellcmd(f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" ' f'{bitbake} {bbtargets} -u toasterui --read {confpath} --read {bblayers} --read {toasterlayers} --token="" >>{log} 2>&1;' - f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'], + f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"', builddir, nowait=True) else: - self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" ' + self._shellcmd(f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" ' f'{local_bitbake} {bbtargets} -u toasterui --token="" >>{log} 2>&1;' - f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'], + f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"', builddir, nowait=True) logger.debug('localhostbecontroller: Build launched, exiting. '