Skip to contentSkip to conversation

Follow-up fixes for command argument conversions

PATCH New Anders Heimer 1 participant 10 messages Jun 25

lore
Share & tools
lore
#0 lore raw

cover Follow-up fixes for command argument conversions

While testing the recent work to replace shell command strings with argument lists I found a few places where behavior previously provided by the shell was not fully represented in argv form. This series contains small, localized corrections which is supposed to restore the previous command semantics.

Anders Heimer (9):
fetch2/az: fix wget command argument construction fetch2/git: avoid an empty argument to git lfs fetch fetch2/git: do not shell-quote the shallow clone remote fetch2/git: keep the gitpkgv commit count numeric fetch2/gitannex: split the annex get arguments fetch2/repo: fix mirror tar command arguments fetch2/sftp: fix port and remote command arguments fetch2/svn: remove shell quotes from the tar exclusion
toaster: preserve shell execution for build launch commands

 lib/bb/fetch2/az.py                             | 10 +++++-----
 lib/bb/fetch2/git.py                            | 10 ++++++----
 lib/bb/fetch2/gitannex.py                       |  2 +-
 lib/bb/fetch2/repo.py                           |  7 +++++--
 lib/bb/fetch2/sftp.py                           |  4 ++--
 lib/bb/fetch2/svn.py                            |  2 +-
 lib/toaster/bldcontrol/localhostbecontroller.py |  8 ++++----
 7 files changed, 24 insertions(+), 19 deletions(-)
#1 lore raw

PATCH 1/9 fetch2/az: fix wget command argument construction

---
 lib/bb/fetch2/az.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
lib/bb/fetch2/az.py +5 -5
index 1d3664f21..81fd14468 100644
@@ -48,17 +48,17 @@ class Az(Wget):
 
         # If were reaching the account transaction limit we might be refused a connection,
         # retrying allows us to avoid false negatives since the limit changes over time
-        fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+        fetchcmd = self.basecmd + ['--retry-connrefused', '--waitretry=5']
 
         # We need to provide a localpath to avoid wget using the SAS
         # ud.localfile either has the downloadfilename or ud.path
         localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
         bb.utils.mkdirhier(os.path.dirname(localpath))
-        fetchcmd += " -O %s" % shlex.quote(localpath)
+        fetchcmd += ["-O", localpath]
 
 
         if ud.user and ud.pswd:
-            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+            fetchcmd += ["--user=%s" % ud.user, "--password=%s" % ud.pswd, "--auth-no-challenge"]
 
         # Check if a Shared Access Signature was given and use it
         az_sas = d.getVar('AZ_SAS')
@@ -73,9 +73,9 @@ class Az(Wget):
         dldir = d.getVar("DL_DIR")
         if os.path.exists(ud.localpath):
             # file exists, but we didnt complete it.. trying again.
-            fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
+            fetchcmd += ["-c", "-P", dldir, azuri]
         else:
-            fetchcmd += " -P %s '%s'" % (dldir, azuri)
+            fetchcmd += ["-P", dldir, azuri]
 
         try:
             self._runwget(ud, d, fetchcmd, False)
#2 lore raw

PATCH 2/9 fetch2/git: avoid an empty argument to git lfs fetch

In reply to Anders Heimer
---
 lib/bb/fetch2/git.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
lib/bb/fetch2/git.py +3 -1
index 0fbd85441..a456465e6 100644
@@ -505,7 +505,9 @@ class Git(FetchMethod):
                 # Using worktree with the revision because .lfsconfig may exists
                 worktree_add_cmd = ud.basecmd + ['worktree', 'add', 'wt', revision]
                 runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
-                lfs_fetch_cmd = ud.basecmd + ['lfs', 'fetch', "--all" if fetchall else ""]
+                lfs_fetch_cmd = ud.basecmd + ['lfs', 'fetch']
+                if fetchall:
+                    lfs_fetch_cmd.append('--all')
                 runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
                 worktree_rem_cmd = ud.basecmd + ['worktree', 'remove', '-f', 'wt']
                 runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
#3 lore raw

PATCH 3/9 fetch2/git: do not shell-quote the shallow clone remote

In reply to Anders Heimer
---
 lib/bb/fetch2/git.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/git.py +1 -1
index a456465e6..00983b744 100644
@@ -580,7 +580,7 @@ class Git(FetchMethod):
         # Use repourl when creating a fast initial shallow clone
         # Prefer already existing full bare clones if available
         if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
-            remote = shlex.quote(repourl)
+            remote = repourl
         else:
             remote = ud.clonedir
         runfetchcmd(ud.basecmd + ['remote', 'add', 'origin', remote], d, workdir=dest)
#4 lore raw

PATCH 4/9 fetch2/git: keep the gitpkgv commit count numeric

In reply to Anders Heimer

The cold-cache path used to keep the raw splitlines() list in commits while only using len(commits) when writing the cache file.

That same commits value is formatted into the returned version string below, so cold cache could return a Python list representation while warm cache returned the numeric count. Store the numeric count in commits before both writing the cache and building the return value.

---
 lib/bb/fetch2/git.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
lib/bb/fetch2/git.py +2 -2
index 00983b744..0bb2ea39a 100644
@@ -1031,9 +1031,9 @@ class Git(FetchMethod):
             commits = None
         else:
             if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
-                commits = bb.fetch2.runfetchcmd(['git', 'rev-list', rev, '--'], d).splitlines()
+                commits = len(bb.fetch2.runfetchcmd(['git', 'rev-list', rev, '--'], d).splitlines())
                 if commits:
-                    open(rev_file, "w").write("%d\n" % len(commits))
+                    open(rev_file, "w").write("%d\n" % commits)
             else:
                 commits = open(rev_file, "r").readline(128).strip()
         if commits:
#5 lore raw

PATCH 5/9 fetch2/gitannex: split the annex get arguments

In reply to Anders Heimer
---
 lib/bb/fetch2/gitannex.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/gitannex.py +1 -1
index b0b2c229e..c378cc36d 100644
@@ -36,7 +36,7 @@ class GitANNEX(Git):
 
     def update_annex(self, ud, d, wd):
         try:
-            runfetchcmd(ud.basecmd + ['annex get', '--all'], d, quiet=True, workdir=wd)
+            runfetchcmd(ud.basecmd + ['annex', 'get', '--all'], d, quiet=True, workdir=wd)
         except bb.fetch.FetchError:
             return False
         runfetchcmd(['chmod', 'u+w', '-R', '%s/annex' % ud.clonedir], d, quiet=True, workdir=wd)
#6 lore raw

PATCH 6/9 fetch2/repo: fix mirror tar command arguments

In reply to Anders Heimer

The direct tar invocation no longer has a shell to expand ./* or strip quotes from exclude patterns. Build the path list explicitly and pass tar exclude patterns without shell quote characters.

---
 lib/bb/fetch2/repo.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
lib/bb/fetch2/repo.py +5 -2
index 8543e3d59..9b96cd824 100644
@@ -73,10 +73,13 @@ class Repo(FetchMethod):
         if scmdata == "keep":
             tar_flags = []
         else:
-            tar_flags = ["--exclude='.repo'", "--exclude='.git'"]
+            tar_flags = ["--exclude=.repo", "--exclude=.git"]
 
         # Create a cache
-        runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath, os.path.join(".", "*")], d, workdir=codir)
+        tar_paths = [os.path.join(".", name)
+                     for name in sorted(os.listdir(codir))
+                     if not name.startswith(".")]
+        runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath] + tar_paths, d, workdir=codir)
 
     def supports_srcrev(self):
         return False
#7 lore raw

PATCH 7/9 fetch2/sftp: fix port and remote command arguments

In reply to Anders Heimer
---
 lib/bb/fetch2/sftp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
lib/bb/fetch2/sftp.py +2 -2
index 959dc6ec2..82c76bb66 100644
@@ -85,7 +85,7 @@ class SFTP(FetchMethod):
         urlo = URI(ud.url)
         basecmd = ['sftp', '-oBatchMode=yes']
         if urlo.port:
-            basecmd += ['-P', urlo.port]
+            basecmd += ['-P', str(urlo.port)]
             urlo.port = None
 
         dldir = d.getVar('DL_DIR')
@@ -102,7 +102,7 @@ class SFTP(FetchMethod):
         if path[:3] == '/~/':
             path = path[3:]
 
-        remote = '"%s%s:%s"' % (user, urlo.hostname, path)
+        remote = '%s%s:%s' % (user, urlo.hostname, path)
 
         cmd = basecmd + [remote, lpath]
 
#8 lore raw

PATCH 8/9 fetch2/svn: remove shell quotes from the tar exclusion

In reply to Anders Heimer
---
 lib/bb/fetch2/svn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
lib/bb/fetch2/svn.py +1 -1
index 84e15c02d..973f76f00 100644
@@ -170,7 +170,7 @@ class Svn(FetchMethod):
             if scmdata == "keep":
                 tar_flags = []
             else:
-                tar_flags = ["--exclude='.svn'"]
+                tar_flags = ["--exclude=.svn"]
 
             # tar them up to a defined filename
             runfetchcmd(['tar'] + tar_flags + ['-czf', ud.localpath, ud.path_spec], d,
#9 lore raw

PATCH 9/9 toaster: preserve shell execution for build launch commands

In reply to Anders Heimer
---
 lib/toaster/bldcontrol/localhostbecontroller.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
lib/toaster/bldcontrol/localhostbecontroller.py +4 -4
index 91c2026f4..4e0a6e90f 100644
@@ -547,14 +547,14 @@ class LocalhostBEController(BuildEnvironmentController):
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
         if not is_merged_attr:
-            self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
+            self._shellcmd(f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
                         f'{bitbake} {bbtargets} -u toasterui  --read {confpath} --read {bblayers} --read {toasterlayers} --token="" >>{log} 2>&1;'
-                        f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
+                        f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"',
                         builddir, nowait=True)
         else:
-            self._shellcmd([f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
+            self._shellcmd(f'{env_clean} bash -c \"(TOASTER_BRBE="{brbe}" BBSERVER="0.0.0.0:{self.be.bbport}" '
                         f'{local_bitbake} {bbtargets} -u toasterui  --token="" >>{log} 2>&1;'
-                        f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"'],
+                        f'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:{self.be.bbport} {bitbake} -m)&\"',
                         builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '