replace os.system/popen with subprocess module
cover replace os.system/popen with subprocess module
These patches are based on oe-core.
* Testinfo
1) $ bitbake core-image-sato core-image-sato-sdk metatoolchain-sdk with MACHINE=qemuarm, PACKAGE_CLASSES = "package_rpm"
2) $ bitbake core-image-sato core-image-sato-sdk metatoolchain-sdk with MACHINE=qemuarm, PACKAGE_CLASSES = "package_ipk"
3) $ bitbake core-image-sato with MACHINE=qemux86, PACKAGE_CLASSES = "package_deb"
4) $ bitbake world with MACHINE=qemux86
// Robert
The following changes since commit a7532d6b2870a51079c39366def9ae55faeba626:
rootfs_rpm.bbclass: save rpmlib rather than remove it (2012-05-29 22:36:39 +0800)
are available in the git repository at:
git://git.pokylinux.org/poky-contrib robert/meta_subprocess
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=robert/meta_subprocess
Robert Yang (4):
meta: replace os.system with subprocess.call
scripts: replace os.system with subprocess.call
meta: replace os.popen with subprocess.Popen
scripts: replace os.popen with subprocess.Popen
meta/classes/archiver.bbclass | 3 +- meta/classes/debian.bbclass | 12 ++++-- meta/classes/distrodata.bbclass | 17 ++++---- meta/classes/icecc.bbclass | 6 +- meta/classes/imagetest-qemu.bbclass | 9 ++-- meta/classes/insane.bbclass | 46 ++++++++++++++------- meta/classes/kernel.bbclass | 6 +- meta/classes/metadata_scm.bbclass | 12 ++--- meta/classes/package.bbclass | 49 ++++++++++++---------- meta/classes/package_deb.bbclass | 3 +- meta/classes/package_ipk.bbclass | 13 ++++-- meta/classes/package_tar.bbclass | 6 ++- meta/classes/sanity.bbclass | 13 +++--- meta/classes/sstate.bbclass | 9 +++- meta/lib/oe/distro_check.py | 3 +- meta/recipes-core/busybox/busybox.inc | 3 +- meta/recipes-core/uclibc/uclibc.inc | 7 ++- meta/recipes-extended/cups/cups14.inc | 3 +- scripts/contrib/python/generate-manifest-2.7.py | 3 +- scripts/rpm-createsolvedb.py | 5 +- scripts/swabber-strace-attach | 3 +- 21 files changed, 136 insertions(+), 95 deletions(-)
PATCH 1/4 meta: replace os.system with subprocess.call
Replace os.system with subprocess.call since the older function would fail (more or less) silently if the executed program cannot be found
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements
[YOCTO #2454]
--- meta/classes/archiver.bbclass | 3 ++- meta/classes/distrodata.bbclass | 5 +++-- meta/classes/imagetest-qemu.bbclass | 9 +++++---- meta/classes/insane.bbclass | 15 ++++++++++----- meta/classes/kernel.bbclass | 4 ++-- meta/classes/package.bbclass | 33 +++++++++++++++++---------------- meta/classes/package_deb.bbclass | 3 ++- meta/classes/package_ipk.bbclass | 13 ++++++++----- meta/classes/package_tar.bbclass | 6 ++++-- meta/classes/sanity.bbclass | 13 +++++++------ meta/classes/sstate.bbclass | 9 ++++++--- meta/lib/oe/distro_check.py | 3 ++- meta/recipes-core/uclibc/uclibc.inc | 7 ++++--- meta/recipes-extended/cups/cups14.inc | 3 ++- 14 files changed, 74 insertions(+), 52 deletions(-)
meta/classes/archiver.bbclass +2 -1
@@ -362,6 +362,7 @@ def dumpdata(d): def create_diff_gz(d): '''creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.g gz for mapping all content in 's' including patches to xxx.diff.gz''' import shutil + import subprocess work_dir = d.getVar('WORKDIR', True) exclude_from = d.getVar('ARCHIVE_EXCLUDE_FROM', True).split()
@@ -387,7 +388,7 @@ def create_diff_gz(d): try: shutil.copy(i, dest) except IOError: - os.system('fakeroot cp -rf ' + i + " " + dest ) + subprocess.call('fakeroot cp -rf ' + i + " " + dest, shell=True) bb.note("Creating .diff.gz in ${DEPLOY_DIR_SRC}/${P}-${PR}.diff.gz") cmd = "LC_ALL=C TZ=UTC0 diff --exclude-from=" + work_dir + "/temp/exclude-from-file -Naur " + s + '.org' + ' ' + s + " | gzip -c > " + diff_file
meta/classes/distrodata.bbclass +3 -2
@@ -231,6 +231,7 @@ python do_checkpkg() { import sys import re import tempfile + import subprocess """ sanity check to ensure same name and type. Match as many patterns as possible
@@ -373,7 +374,7 @@ python do_checkpkg() { f.close() if status != "ErrHostNoDir" and re.match("Err", status): logpath = d.getVar('LOG_DIR', True) - os.system("cp %s %s/" % (f.name, logpath)) + subprocess.call("cp %s %s/" % (f.name, logpath), shell=True) os.unlink(f.name) return status
@@ -432,7 +433,7 @@ python do_checkpkg() { """if host hasn't directory information, no need to save tmp file""" if status != "ErrHostNoDir" and re.match("Err", status): logpath = d.getVar('LOG_DIR', True) - os.system("cp %s %s/" % (f.name, logpath)) + subprocess.call("cp %s %s/" % (f.name, logpath), shell=True) os.unlink(f.name) return status
meta/classes/imagetest-qemu.bbclass +5 -4
@@ -28,6 +28,7 @@ def qemuimagetest_main(d): import re import os import shutil + import subprocess """ Test Controller for automated testing.
@@ -58,7 +59,7 @@ def qemuimagetest_main(d): logpath = d.getVar('TEST_LOG', True) bb.utils.mkdirhier("%s/%s" % (logpath, scen)) caselog = os.path.join(logpath, "%s/log_%s.%s" % (scen, case, d.getVar('DATETIME', True))) - os.system("touch %s" % caselog) + subprocess.call("touch %s" % caselog, shell=True) """export TEST_TMP, TEST_RESULT, DEPLOY_DIR and QEMUARCH""" os.environ["PATH"] = d.getVar("PATH", True)
@@ -78,7 +79,7 @@ def qemuimagetest_main(d): """run Test Case""" bb.note("Run %s test in scenario %s" % (case, scen)) - os.system("%s" % fulltestpath) + subprocess.call("%s" % fulltestpath, shell=True) """function to check testcase list and remove inappropriate cases""" def check_list(list):
@@ -168,7 +169,7 @@ def qemuimagetest_main(d): test_status = d.getVar('TEST_STATUS', True) if os.path.exists(test_status): os.remove(test_status) - os.system("touch %s" % test_status) + subprocess.call("touch %s" % test_status, shell=True) """initialize result file""" resultpath = d.getVar('TEST_RESULT', True)
@@ -180,7 +181,7 @@ def qemuimagetest_main(d): if os.path.exists(sresultfile): os.remove(sresultfile) - os.system("touch %s" % resultfile) + subprocess.call("touch %s" % resultfile, shell=True) os.symlink(resultfile, sresultfile) f = open(sresultfile, "a") f.write("\tTest Result for %s %s\n" % (machine, pname))
meta/classes/insane.bbclass +10 -5
@@ -267,6 +267,7 @@ def package_qa_check_unsafe_references_in_scripts(path, name, d, elf, messages): if not elf: import stat + import subprocess pn = d.getVar('PN', True) # Ensure we're checking an executable script
@@ -275,7 +276,7 @@ def package_qa_check_unsafe_references_in_scripts(path, name, d, elf, messages): # grep shell scripts for possible references to /exec_prefix/ exec_prefix = d.getVar('exec_prefix', True) statement = "grep -e '%s/' %s > /dev/null" % (exec_prefix, path) - if os.system(statement) == 0: + if subprocess.call(statement, shell=True) == 0: error_msg = pn + ": Found a reference to %s/ in %s" % (exec_prefix, path) package_qa_handle_error("unsafe-references-in-scripts", error_msg, d) error_msg = "Shell scripts in base_bindir and base_sbindir should not reference anything in exec_prefix"
@@ -609,6 +610,8 @@ def package_qa_check_rdepends(pkg, pkgdest, skip, d): # The PACKAGE FUNC to scan each package python do_package_qa () { + import subprocess + bb.note("DO PACKAGE QA") logdir = d.getVar('T', True)
@@ -619,7 +622,7 @@ python do_package_qa () { if os.path.exists(compilelog): statement = "grep -e 'CROSS COMPILE Badness:' -e 'is unsafe for cross-compilation' %s > /dev/null" % compilelog - if os.system(statement) == 0: + if subprocess.call(statement, shell=True) == 0: bb.warn("%s: The compile log indicates that host include and/or library paths were used.\n \ Please check the log '%s' for more information." % (pkg, compilelog))
@@ -628,7 +631,7 @@ python do_package_qa () { if os.path.exists(installlog): statement = "grep -e 'CROSS COMPILE Badness:' -e 'is unsafe for cross-compilation' %s > /dev/null" % installlog - if os.system(statement) == 0: + if subprocess.call(statement, shell=True) == 0: bb.warn("%s: The install log indicates that host include and/or library paths were used.\n \ Please check the log '%s' for more information." % (pkg, installlog))
@@ -684,6 +687,8 @@ python do_qa_staging() { } python do_qa_configure() { + import subprocess + configs = [] workdir = d.getVar('WORKDIR', True) bb.note("Checking autotools environment for common misconfiguration")
@@ -691,7 +696,7 @@ python do_qa_configure() { statement = "grep -e 'CROSS COMPILE Badness:' -e 'is unsafe for cross-compilation' %s > /dev/null" % \ os.path.join(root,"config.log") if "config.log" in files: - if os.system(statement) == 0: + if subprocess.call(statement, shell=True) == 0: bb.fatal("""This autoconf log indicates errors, it looked at host include and/or library paths while determining system capabilities. Rerun configure task after fixing this. The path was '%s'""" % root)
@@ -713,7 +718,7 @@ Rerun configure task after fixing this. The path was '%s'""" % root) if gt not in deps: for config in configs: gnu = "grep \"^[[:space:]]*AM_GNU_GETTEXT\" %s >/dev/null" % config - if os.system(gnu) == 0: + if subprocess.call(gnu, shell=True) == 0: bb.fatal("""%s required but not in DEPENDS for file %s. Missing inherit gettext?""" % (gt, config))
meta/classes/kernel.bbclass +2 -2
@@ -313,12 +313,12 @@ module_conf_rfcomm = "alias bt-proto-3 rfcomm" python populate_packages_prepend () { def extract_modinfo(file): - import tempfile, re + import tempfile, re, subprocess tempfile.tempdir = d.getVar("WORKDIR", True) tf = tempfile.mkstemp() tmpfile = tf[1] cmd = "PATH=\"%s\" %sobjcopy -j .modinfo -O binary %s %s" % (d.getVar("PATH", True), d.getVar("HOST_PREFIX", True) or "", file, tmpfile) - os.system(cmd) + subprocess.call(cmd, shell=True) f = open(tmpfile) l = f.read().split("\000") f.close()
meta/classes/package.bbclass +17 -16
@@ -183,7 +183,7 @@ def splitfile(file, debugfile, debugsrcdir, d): # The debug information is then processed for src references. These # references are copied to debugsrcdir, if defined. - import commands, stat + import commands, stat, subprocess dvar = d.getVar('PKGD', True) pathprefix = "export PATH=%s; " % d.getVar('PATH', True)
@@ -205,14 +205,14 @@ def splitfile(file, debugfile, debugsrcdir, d): # We need to extract the debug src information here... if debugsrcdir: - os.system("%s'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (pathprefix, debugedit, workparentdir, debugsrcdir, sourcefile, file)) + subprocess.call("%s'%s' -b '%s' -d '%s' -i -l '%s' '%s'" % (pathprefix, debugedit, workparentdir, debugsrcdir, sourcefile, file), shell=True) bb.mkdirhier(os.path.dirname(debugfile)) - os.system("%s'%s' --only-keep-debug '%s' '%s'" % (pathprefix, objcopy, file, debugfile)) + subprocess.call("%s'%s' --only-keep-debug '%s' '%s'" % (pathprefix, objcopy, file, debugfile), shell=True) # Set the debuglink to have the view of the file path on the target - os.system("%s'%s' --add-gnu-debuglink='%s' '%s'" % (pathprefix, objcopy, debugfile, file)) + subprocess.call("%s'%s' --add-gnu-debuglink='%s' '%s'" % (pathprefix, objcopy, debugfile, file), shell=True) if newmode: os.chmod(file, origmode)
@@ -225,7 +225,7 @@ def splitfile2(debugsrcdir, d): # The debug src information processed in the splitfile2 is further procecessed # and copied to the destination here. - import commands, stat + import commands, stat, subprocess sourcefile = d.expand("${WORKDIR}/debugsources.list") if debugsrcdir and os.path.isfile(sourcefile):
@@ -252,14 +252,14 @@ def splitfile2(debugsrcdir, d): processdebugsrc += "fgrep -z '%s' | " processdebugsrc += "(cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s' 2>/dev/null)" - os.system(processdebugsrc % (sourcefile, workbasedir, workparentdir, dvar, debugsrcdir)) + subprocess.call(processdebugsrc % (sourcefile, workbasedir, workparentdir, dvar, debugsrcdir), shell=True) # The copy by cpio may have resulted in some empty directories! Remove these for root, dirs, files in os.walk("%s%s" % (dvar, debugsrcdir)): for d in dirs: dir = os.path.join(root, d) #bb.note("rmdir -p %s" % dir) - os.system("rmdir -p %s 2>/dev/null" % dir) + subprocess.call("rmdir -p %s 2>/dev/null" % dir, shell=True) # Also remove debugsrcdir if its empty for p in nosuchdir[::-1]:
@@ -275,14 +275,14 @@ def runstrip(file, elftype, d): # 4 - executable # 8 - shared library - import commands, stat + import commands, stat, subprocess pathprefix = "export PATH=%s; " % d.getVar('PATH', True) strip = d.getVar("STRIP", True) # Handle kernel modules specifically - .debug directories here are pointless if file.find("/lib/modules/") != -1 and file.endswith(".ko"): - return os.system("%s'%s' --strip-debug --remove-section=.comment --remove-section=.note --preserve-dates '%s'" % (pathprefix, strip, file)) + return subprocess.call("%s'%s' --strip-debug --remove-section=.comment --remove-section=.note --preserve-dates '%s'" % (pathprefix, strip, file), shell=True) newmode = None if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
@@ -301,7 +301,7 @@ def runstrip(file, elftype, d): stripcmd = "'%s' %s '%s'" % (strip, extraflags, file) bb.debug(1, "runstrip: %s" % stripcmd) - ret = os.system("%s%s" % (pathprefix, stripcmd)) + ret = subprocess.call("%s%s" % (pathprefix, stripcmd), shell=True) if newmode: os.chmod(file, origmode)
@@ -427,6 +427,7 @@ python package_do_split_locales() { } python perform_packagecopy () { + import subprocess dest = d.getVar('D', True) dvar = d.getVar('PKGD', True)
@@ -434,9 +435,9 @@ python perform_packagecopy () { # Start by package population by taking a copy of the installed # files to operate on - os.system('rm -rf %s/*' % (dvar)) + subprocess.call('rm -rf %s/*' % (dvar), shell=True) # Preserve sparse files and hard links - os.system('tar -cf - -C %s -ps . | tar -xf - -C %s' % (dest, dvar)) + subprocess.call('tar -cf - -C %s -ps . | tar -xf - -C %s' % (dest, dvar), shell=True) } # We generate a master list of directories to process, we start by
@@ -668,7 +669,7 @@ python fixup_perms () { } python split_and_strip_files () { - import commands, stat, errno + import commands, stat, errno, subprocess dvar = d.getVar('PKGD', True) pn = d.getVar('PN', True)
@@ -838,7 +839,7 @@ python split_and_strip_files () { os.unlink(fpath) # This could leave an empty debug directory laying around # take care of the obvious case... - os.system("rmdir %s 2>/dev/null" % os.path.dirname(fpath)) + subprocess.call("rmdir %s 2>/dev/null" % os.path.dirname(fpath), shell=True) # Process the debugsrcdir if requested... # This copies and places the referenced sources for later debugging...
@@ -870,7 +871,7 @@ python split_and_strip_files () { } python populate_packages () { - import glob, stat, errno, re + import glob, stat, errno, re, subprocess workdir = d.getVar('WORKDIR', True) outdir = d.getVar('DEPLOY_DIR', True)
@@ -896,7 +897,7 @@ python populate_packages () { package_list.append(pkg) d.setVar('PACKAGES', ' '.join(package_list)) pkgdest = d.getVar('PKGDEST', True) - os.system('rm -rf %s' % pkgdest) + subprocess.call('rm -rf %s' % pkgdest, shell=True) seen = []
meta/classes/package_deb.bbclass +2 -1
@@ -205,6 +205,7 @@ deb_log_check() { python do_package_deb () { import re, copy import textwrap + import subprocess workdir = d.getVar('WORKDIR', True) if not workdir:
@@ -384,7 +385,7 @@ python do_package_deb () { conffiles.close() os.chdir(basedir) - ret = os.system("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH", True), root, pkgoutdir)) + ret = subprocess.call("PATH=\"%s\" dpkg-deb -b %s %s" % (localdata.getVar("PATH", True), root, pkgoutdir), shell=True) if ret != 0: bb.utils.prunedir(controldir) bb.utils.unlockfile(lf)
meta/classes/package_ipk.bbclass +8 -5
@@ -15,6 +15,8 @@ python package_ipk_fn () { } python package_ipk_install () { + import subprocess + pkg = d.getVar('PKG', True) pkgfn = d.getVar('PKGFN', True) rootfs = d.getVar('IMAGE_ROOTFS', True)
@@ -52,14 +54,14 @@ python package_ipk_install () { if not os.access(os.path.join(ipkdir,"Packages"), os.R_OK) or not os.access(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"),os.R_OK): - ret = os.system('opkg-make-index -p %s %s ' % (os.path.join(ipkdir, "Packages"), ipkdir)) + ret = subprocess.call('opkg-make-index -p %s %s ' % (os.path.join(ipkdir, "Packages"), ipkdir), shell=True) if (ret != 0 ): raise bb.build.FuncFailed f = open(os.path.join(tmpdir, "stamps", "IPK_PACKAGE_INDEX_CLEAN"),"w") f.close() - ret = os.system('opkg-cl -o %s -f %s update' % (rootfs, conffile)) - ret = os.system('opkg-cl -o %s -f %s install %s' % (rootfs, conffile, pkgfn)) + ret = subprocess.call('opkg-cl -o %s -f %s update' % (rootfs, conffile), shell=True) + ret = subprocess.call('opkg-cl -o %s -f %s install %s' % (rootfs, conffile, pkgfn), shell=True) if (ret != 0 ): raise bb.build.FuncFailed }
@@ -262,6 +264,7 @@ package_generate_archlist () { python do_package_ipk () { import re, copy import textwrap + import subprocess workdir = d.getVar('WORKDIR', True) outdir = d.getVar('PKGWRITEDIRIPK', True)
@@ -419,8 +422,8 @@ python do_package_ipk () { conffiles.close() os.chdir(basedir) - ret = os.system("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH", True), - d.getVar("OPKGBUILDCMD",1), pkg, pkgoutdir)) + ret = subprocess.call("PATH=\"%s\" %s %s %s" % (localdata.getVar("PATH", True), + d.getVar("OPKGBUILDCMD",1), pkg, pkgoutdir), shell=True) if ret != 0: bb.utils.unlockfile(lf) raise bb.build.FuncFailed("opkg-build execution failed")
meta/classes/package_tar.bbclass +4 -2
@@ -9,6 +9,7 @@ python package_tar_fn () { } python package_tar_install () { + import subprocess pkg = d.getVar('PKG', True) pkgfn = d.getVar('PKGFN', True) rootfs = d.getVar('IMAGE_ROOTFS', True)
@@ -29,12 +30,13 @@ python package_tar_install () { bb.debug(1, "%s does not exist, skipping" % pkgfn) raise bb.build.FuncFailed - ret = os.system('zcat %s | tar -xf -' % pkgfn) + ret = subprocess.call('zcat %s | tar -xf -' % pkgfn, shell=True) if ret != 0: raise bb.build.FuncFailed } python do_package_tar () { + import subprocess workdir = d.getVar('WORKDIR', True) if not workdir: bb.error("WORKDIR not defined, unable to package")
@@ -85,7 +87,7 @@ python do_package_tar () { if not glob('*'): bb.note("Not creating empty archive for %s-%s-%s" % (pkg, localdata.getVar('PKGV', True), localdata.getVar('PKGR', True))) continue - ret = os.system("tar -czf %s %s" % (tarfn, '.')) + ret = subprocess.call("tar -czf %s %s" % (tarfn, '.'), shell=True) if ret != 0: bb.error("Creation of tar %s failed." % tarfn) }
meta/classes/sanity.bbclass +7 -6
@@ -282,6 +282,7 @@ def check_sanity_validmachine(sanity_data): def check_sanity(sanity_data): from bb import note, error, data, __version__ + import subprocess try: from distutils.version import LooseVersion
@@ -495,16 +496,16 @@ def check_sanity(sanity_data): f.write(current_abi) elif abi == "2" and current_abi == "3": bb.note("Converting staging from layout version 2 to layout version 3") - os.system(sanity_data.expand("mv ${TMPDIR}/staging ${TMPDIR}/sysroots")) - os.system(sanity_data.expand("ln -s sysroots ${TMPDIR}/staging")) - os.system(sanity_data.expand("cd ${TMPDIR}/stamps; for i in */*do_populate_staging; do new=`echo $i | sed -e 's/do_populate_staging/do_populate_sysroot/'`; mv $i $new; done")) + subprocess.call(sanity_data.expand("mv ${TMPDIR}/staging ${TMPDIR}/sysroots"), shell=True) + subprocess.call(sanity_data.expand("ln -s sysroots ${TMPDIR}/staging"), shell=True) + subprocess.call(sanity_data.expand("cd ${TMPDIR}/stamps; for i in */*do_populate_staging; do new=`echo $i | sed -e 's/do_populate_staging/do_populate_sysroot/'`; mv $i $new; done"), shell=True) f = file(abifile, "w") f.write(current_abi) elif abi == "3" and current_abi == "4": bb.note("Converting staging layout from version 3 to layout version 4") if os.path.exists(sanity_data.expand("${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}")): - os.system(sanity_data.expand("mv ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS} ${STAGING_BINDIR_CROSS}")) - os.system(sanity_data.expand("ln -s ${STAGING_BINDIR_CROSS} ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}")) + subprocess.call(sanity_data.expand("mv ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS} ${STAGING_BINDIR_CROSS}"), shell=True) + subprocess.call(sanity_data.expand("ln -s ${STAGING_BINDIR_CROSS} ${STAGING_DIR_NATIVE}${bindir_native}/${MULTIMACH_HOST_SYS}"), shell=True) f = file(abifile, "w") f.write(current_abi)
@@ -512,7 +513,7 @@ def check_sanity(sanity_data): messages = messages + "Staging layout has changed. The cross directory has been deprecated and cross packages are now built under the native sysroot.\nThis requires a rebuild.\n" elif abi == "5" and current_abi == "6": bb.note("Converting staging layout from version 5 to layout version 6") - os.system(sanity_data.expand("mv ${TMPDIR}/pstagelogs ${SSTATE_MANIFESTS}")) + subprocess.call(sanity_data.expand("mv ${TMPDIR}/pstagelogs ${SSTATE_MANIFESTS}"), shell=True) f = file(abifile, "w") f.write(current_abi) elif abi == "7" and current_abi == "8":
meta/classes/sstate.bbclass +6 -3
@@ -145,6 +145,7 @@ def sstate_install(ss, d): def sstate_installpkg(ss, d): import oe.path + import subprocess def prepdir(dir): # remove dir if it exists, ensure any parent directories do exist
@@ -195,7 +196,7 @@ def sstate_installpkg(ss, d): sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd) print "Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd) - os.system(sstate_hardcode_cmd) + subprocess.call(sstate_hardcode_cmd, shell=True) # Need to remove this or we'd copy it into the target directory and may # conflict with another writer
@@ -309,6 +310,8 @@ python sstate_cleanall() { } def sstate_hardcode_path(d): + import subprocess + # Need to remove hardcoded paths and fix these when we install the # staging packages. #
@@ -343,14 +346,14 @@ def sstate_hardcode_path(d): sstate_hardcode_cmd = "%s | xargs %s | %s | xargs --no-run-if-empty %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, sstate_sed_cmd) print "Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd) - os.system(sstate_hardcode_cmd) + subprocess.call(sstate_hardcode_cmd, shell=True) # If the fixmefn is empty, remove it.. if os.stat(fixmefn).st_size == 0: os.remove(fixmefn) else: print "Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd) - os.system(sstate_filelist_relative_cmd) + subprocess.call(sstate_filelist_relative_cmd, shell=True) def sstate_package(ss, d): import oe.path
meta/lib/oe/distro_check.py +2 -1
@@ -343,6 +343,7 @@ def compare_in_distro_packages_list(distro_check_dir, d): return matching_distros def create_log_file(d, logname): + import subprocess logpath = d.getVar('LOG_DIR', True) bb.utils.mkdirhier(logpath) logfn, logsuffix = os.path.splitext(logname)
@@ -351,7 +352,7 @@ def create_log_file(d, logname): slogfile = os.path.join(logpath, logname) if os.path.exists(slogfile): os.remove(slogfile) - os.system("touch %s" % logfile) + subprocess.call("touch %s" % logfile, shell=True) os.symlink(logfile, slogfile) d.setVar('LOG_FILE', logfile) return logfile
meta/recipes-core/uclibc/uclibc.inc +4 -3
@@ -167,9 +167,10 @@ python () { "/^### ABI$/a\\\nCONFIG_%s=y\n\n" % ("${UCLIBC_ABI}")) } -do_patch_append() { - os.system("ln -sf ${STAGING_INCDIR}/linux ${S}/include/linux") - os.system("ln -sf ${STAGING_INCDIR}/asm ${S}/include/asm") +python do_patch_append() { + import subprocess + subprocess.call("ln -sf ${STAGING_INCDIR}/linux ${S}/include/linux", shell=True) + subprocess.call("ln -sf ${STAGING_INCDIR}/asm ${S}/include/asm", shell=True) } do_configure() {
meta/recipes-extended/cups/cups14.inc +2 -1
@@ -57,9 +57,10 @@ fakeroot do_install () { } python do_package_append() { + import subprocess # Change permissions back the way they were, they probably had a reason... workdir = d.getVar('WORKDIR', True) - os.system('chmod 0511 %s/install/cups/var/run/cups/certs' % workdir) + subprocess.call('chmod 0511 %s/install/cups/var/run/cups/certs' % workdir, shell=True) } PACKAGES =+ "${PN}-lib ${PN}-libimage"
Signature · 1 line
1.7.1
PATCH 2/4 scripts: replace os.system with subprocess.call
In reply to Robert YangReplace os.system with subprocess.call since the older function would fail (more or less) silently if the executed program cannot be found
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements
[YOCTO #2454]
--- scripts/rpm-createsolvedb.py | 5 +++-- scripts/swabber-strace-attach | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-)
scripts/rpm-createsolvedb.py +3 -2
@@ -14,6 +14,7 @@ import sys, os import hashlib import stat +import subprocess if len(sys.argv) < 1: print("Error, rpm command not specified")
@@ -44,7 +45,7 @@ for path in paths: continue if os.path.exists(path + "/solvedb"): - os.system("rm -rf %s" % (path + "/solvedb")) + subprocess.call("rm -rf %s" % (path + "/solvedb"), shell=True) os.mkdir(path + "/solvedb") m = open(path + "/solvedb/manifest", "w") m.write("# Dynamically generated solve manifest\n")
@@ -56,7 +57,7 @@ for path in paths: --noaid --nodeps --noorder --noscripts --notriggers --noparentdirs --nolinktos --stats \ --ignoresize --nosignature --nodigest -D "__dbi_txn create nofsync" \ ' + path + '/solvedb/manifest' - os.system(cmd) + subprocess.call(cmd, shell=True) open(path + "/solvedb.checksum", "w").write(checksum) open(path + "/solvedb.done", "w")
scripts/swabber-strace-attach +2 -1
@@ -1,6 +1,7 @@ #!/usr/bin/env python import os import sys +import subprocess # Detach from the controlling terminal and parent process by forking twice to daemonize ourselves, # then run the command passed as argv[1]. Send log data to argv[2].
@@ -24,7 +25,7 @@ os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) -ret = os.system(sys.argv[1]) +ret = subprocess.call(sys.argv[1], shell=True) os._exit(ret)
Signature · 1 line
1.7.1
PATCH 3/4 meta: replace os.popen with subprocess.Popen
In reply to Robert YangReplace os.popen with subprocess.Popen since the older function would fail (more or less) silently if the executed program cannot be found
There are both bb.process.run() and bb.process.Popen() which wraps the subprocess module, use it for simplifying the code.
Note: We don't need the "2>/dev/null" or "2>&1" since bb.process.run() can handle it, it will raise exception when error occurs, we should handle the exception ourselves if we want to ignore the error.
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements
[YOCTO #2454]
--- meta/classes/debian.bbclass | 12 ++++++++---- meta/classes/distrodata.bbclass | 12 ++++++------ meta/classes/icecc.bbclass | 6 +++--- meta/classes/insane.bbclass | 31 ++++++++++++++++++++----------- meta/classes/kernel.bbclass | 2 +- meta/classes/metadata_scm.bbclass | 12 +++++------- meta/classes/package.bbclass | 16 ++++++++++------ meta/recipes-core/busybox/busybox.inc | 3 ++- 8 files changed, 55 insertions(+), 39 deletions(-)
meta/classes/debian.bbclass +8 -4
@@ -60,10 +60,14 @@ python debian_package_name_hook () { for f in files: if so_re.match(f): fp = os.path.join(root, f) - cmd = (d.getVar('BUILD_PREFIX', True) or "") + "objdump -p " + fp + " 2>/dev/null" - fd = os.popen(cmd) - lines = fd.readlines() - fd.close() + cmd = (d.getVar('BUILD_PREFIX', True) or "") + "objdump -p " + fp + try: + lines = "" + lines = bb.process.run(cmd)[0] + # Some ".so" maybe ascii text, e.g: /usr/lib64/libpthread.so, + # ingore those errors. + except Exception: + sys.exc_clear() for l in lines: m = re.match("\s+SONAME\s+([^\s]*)", l) if m and not m.group(1) in sonames:
meta/classes/distrodata.bbclass +6 -6
@@ -564,10 +564,10 @@ python do_checkpkg() { gitproto = parm['protocol'] else: gitproto = "git" - gitcmd = "git ls-remote %s://%s%s%s *tag* 2>&1" % (gitproto, gituser, host, path) - gitcmd2 = "git ls-remote %s://%s%s%s HEAD 2>&1" % (gitproto, gituser, host, path) - tmp = os.popen(gitcmd).read() - tmp2 = os.popen(gitcmd2).read() + gitcmd = "git ls-remote %s://%s%s%s *tag*" % (gitproto, gituser, host, path) + gitcmd2 = "git ls-remote %s://%s%s%s HEAD" % (gitproto, gituser, host, path) + tmp = bb.process.run(gitcmd)[0] + tmp2 = bb.process.run(gitcmd2)[0] #This is for those repo have tag like: refs/tags/1.2.2 if tmp: tmpline = tmp.split("\n")
@@ -613,9 +613,9 @@ python do_checkpkg() { if 'rev' in parm: pcurver = parm['rev'] - svncmd = "svn info %s %s://%s%s/%s/ 2>&1" % (" ".join(options), svnproto, host, path, parm["module"]) + svncmd = "svn info %s %s://%s%s/%s/" % (" ".join(options), svnproto, host, path, parm["module"]) print svncmd - svninfo = os.popen(svncmd).read() + svninfo = bb.process.run(svncmd)[0] for line in svninfo.split("\n"): if re.search("^Last Changed Rev:", line): pupver = line.split(" ")[-1]
meta/classes/icecc.bbclass +3 -3
@@ -54,7 +54,7 @@ def create_path(compilers, bb, d): staging += "-kernel" #check if the icecc path is set by the user - icecc = d.getVar('ICECC_PATH') or os.popen("which icecc").read()[:-1] + icecc = d.getVar('ICECC_PATH') or bb.process.run("which icecc")[0][:-1] # Create the dir if necessary try:
@@ -151,9 +151,9 @@ def icc_path(bb,d): def icc_get_tool(bb, d, tool): if icc_is_native(bb, d): - return os.popen("which %s" % tool).read()[:-1] + return bb.process.run("which %s" % tool)[0][:-1] elif icc_is_kernel(bb, d): - return os.popen("which %s" % get_cross_kernel_cc(bb, d)).read()[:-1] + return bb.process.run("which %s" % get_cross_kernel_cc(bb, d))[0][:-1] else: ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}') target_sys = d.expand('${TARGET_SYS}')
meta/classes/insane.bbclass +20 -11
@@ -154,14 +154,29 @@ def package_qa_check_rpath(file,name, d, elf, messages): if not bad_dirs[0] in d.getVar('WORKDIR', True): bb.fatal("This class assumed that WORKDIR is ${TMPDIR}/work... Not doing any check") - output = os.popen("%s -B -F%%r#F '%s'" % (scanelf,file)) - txt = output.readline().split() + output, errors = bb.process.run("%s -B -F%%r#F '%s'" % (scanelf,file)) + txt = output.split() for line in txt: for dir in bad_dirs: if dir in line: messages.append("package %s contains bad RPATH %s in file %s" % (name, line, file)) QAPATHTEST[useless-rpaths] = "package_qa_check_useless_rpaths" + +def package_qa_get_objdump(d, path): + """ + Get the result of objdump, ignore the errors since not all files can be objdumped + """ + env_path = d.getVar('PATH', True) + objdump = d.getVar('OBJDUMP', True) + + try: + lines = "" + lines = bb.process.run("LC_ALL=C PATH=%s %s -p '%s'" % (env_path, objdump, path))[0] + except Exception: + sys.exc_clear() + return lines + def package_qa_check_useless_rpaths(file, name, d, elf, messages): """ Check for RPATHs that are useless but not dangerous
@@ -169,15 +184,12 @@ def package_qa_check_useless_rpaths(file, name, d, elf, messages): if not elf: return - objdump = d.getVar('OBJDUMP', True) - env_path = d.getVar('PATH', True) - libdir = d.getVar("libdir", True) base_libdir = d.getVar("base_libdir", True) import re rpath_re = re.compile("\s+RPATH\s+(.*)") - for line in os.popen("LC_ALL=C PATH=%s %s -p '%s' 2> /dev/null" % (env_path, objdump, file), "r"): + for line in package_qa_get_objdump(d, file): m = rpath_re.match(line) if m: rpath = m.group(1)
@@ -369,7 +381,7 @@ def package_qa_check_desktop(path, name, d, elf, messages): """ if path.endswith(".desktop"): desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE',True),'desktop-file-validate') - output = os.popen("%s %s" % (desktop_file_validate, path)) + output, errors = bb.process.run("%s %s" % (desktop_file_validate, path)) # This only produces output on errors for l in output: messages.append("Desktop file issue: " + l.strip())
@@ -392,14 +404,11 @@ def package_qa_hash_style(path, name, d, elf, messages): if not gnu_hash: return - objdump = d.getVar('OBJDUMP', True) - env_path = d.getVar('PATH', True) - sane = False has_syms = False # If this binary has symbols, we expect it to have GNU_HASH too. - for line in os.popen("LC_ALL=C PATH=%s %s -p '%s' 2> /dev/null" % (env_path, objdump, path), "r"): + for line in package_qa_get_objdump(d, path): if "SYMTAB" in line: has_syms = True if "GNU_HASH" in line:
meta/classes/kernel.bbclass +1 -1
@@ -349,7 +349,7 @@ python populate_packages_prepend () { path = d.getVar("PATH", True) cmd = "PATH=\"%s\" depmod -n -a -b %s -F %s/boot/System.map-%s %s" % (path, dvar, dvar, kernelver, kernelver_stripped) - f = os.popen(cmd, 'r') + f = bb.process.Popen(cmd, shell=True).stdout deps = {} pattern0 = "^(.*\.k?o):..*$"
meta/classes/metadata_scm.bbclass +5 -7
@@ -60,18 +60,16 @@ def base_get_metadata_svn_revision(path, d): return revision def base_get_metadata_git_branch(path, d): - branch = os.popen('cd %s; git branch 2>&1 | grep "^* " | tr -d "* "' % path).read() + branch = bb.process.run('cd %s; git branch | grep "^* " | tr -d "* "' % path)[0] if len(branch) != 0: return branch return "<unknown>" def base_get_metadata_git_revision(path, d): - f = os.popen("cd %s; git log -n 1 --pretty=oneline -- 2>&1" % path) - data = f.read() - if f.close() is None: - rev = data.split(" ")[0] - if len(rev) != 0: - return rev + rev = bb.process.run("cd %s; git log -n 1 --pretty=oneline" % path)[0] + if len(rev) != 0: + rev = rev.split(" ")[0] + return rev return "<unknown>"
meta/classes/package.bbclass +10 -6
@@ -1061,7 +1061,7 @@ python emit_pkgdata() { def get_directory_size(dir): if os.listdir(dir): - size = int(os.popen('du -sk %s' % dir).readlines()[0].split('\t')[0]) + size = int(bb.process.run('du -sk %s' % dir)[0].split('\t')[0]) else: size = 0 return size
@@ -1217,7 +1217,7 @@ python package_do_filedeps() { rpfiles.append(os.path.join(root, file)) for files in chunks(rpfiles, 100): - dep_pipe = os.popen(rpmdeps + " " + " ".join(files)) + dep_pipe = bb.process.Popen(rpmdeps + " " + " ".join(files), shell=True).stdout process_deps(dep_pipe, pkg, provides_files, requires_files)
@@ -1259,11 +1259,15 @@ python package_do_shlibs() { def linux_so(root, path, file): needs_ldconfig = False - cmd = d.getVar('OBJDUMP', True) + " -p " + pipes.quote(os.path.join(root, file)) + " 2>/dev/null" + cmd = d.getVar('OBJDUMP', True) + " -p " + pipes.quote(os.path.join(root, file)) cmd = "PATH=\"%s\" %s" % (d.getVar('PATH', True), cmd) - fd = os.popen(cmd) - lines = fd.readlines() - fd.close() + try: + lines = "" + lines = bb.process.run(cmd)[0] + # Some ".so" maybe ascii text, e.g: /usr/lib64/libpthread.so, + # ingore those errors. + except Exception: + sys.exc_clear() for l in lines: m = re.match("\s+NEEDED\s+([^\s]*)", l) if m:
meta/recipes-core/busybox/busybox.inc +2 -1
@@ -215,7 +215,8 @@ python package_do_filedeps_append () { # Load/backup original set filerprovides = d.getVar('FILERPROVIDES_%s_%s' % (f_busybox, pkg), True) or "" - dep_pipe = os.popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links)) + dep_pipe = bb.process.Popen('sed -e "s,^,%s/%s%s Provides: ," %s/%s%s' % \ + (pkgdest, pkg, f_busybox, pkgdest, pkg, f_busybox_links), shell=True).stdout process_deps(dep_pipe, pkg, provides_files, requires_files)
Signature · 1 line
1.7.1
PATCH 4/4 scripts: replace os.popen with subprocess.Popen
In reply to Robert YangReplace os.popen with subprocess.Popen since the older function would fail (more or less) silently if the executed program cannot be found
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements
[YOCTO #2454]
--- scripts/contrib/python/generate-manifest-2.7.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
scripts/contrib/python/generate-manifest-2.7.py +2 -1
@@ -13,6 +13,7 @@ import os import sys import time +import subprocess VERSION = "2.7.2"
@@ -149,7 +150,7 @@ class MakefileMaker: if __name__ == "__main__": if len( sys.argv ) > 1: - os.popen( "rm -f ./%s" % sys.argv[1] ) + subprocess.Popen("rm -f ./%s" % sys.argv[1], shell=True, stdout=subprocess.PIPE).stdout outfile = file( sys.argv[1], "w" ) else: outfile = sys.stdout
Signature · 1 line
1.7.1
Re: [PATCH 4/4] scripts: replace os.popen with subprocess.Popen
Quoted 28 lines · Robert Yang
Replace os.popen with subprocess.Popen since the older function would fail (more or less) silently if the executed program cannot be found
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements[YOCTO #2454]
Signed-off-by Robert Yang <[email protected]>---
scripts/contrib/python/generate-manifest-2.7.py | 3 ++-1 files changed, 2 insertions(+), 1 deletions(-)scripts/contrib/python/generate-manifest-2.7.py +0
-0@@ -13,6 +13,7 @@import os
import sys
import time
+import subprocessVERSION = "2.7.2"
scripts/contrib/python/generate-manifest-2.7.py +0
-0@@ -149,7 +150,7 @@ class MakefileMaker:if
__name__== "__main__":
Quoted 6 lines
if len( sys.argv ) > 1:
- os.popen( "rm -f ./%s" % sys.argv[1] )
+ subprocess.Popen("rm -f ./%s" % sys.argv[1], shell=True, stdout=subprocess.PIPE).stdout outfile =file( sys.argv[1], "w" )
else:
outfile = sys.stdout
This seems like a candidate for subprocess.call, not subprocess.Popen, as the output isn't used. Better yet, just use os.unlink() and ignore exceptions from it.
Signature · 1 line
Christopher Larson
Re: [PATCH 4/4] scripts: replace os.popen with subprocess.Popen
Quoted 29 lines · Chris Larson
Quoted 28 lines · Robert Yang
Replace os.popen with subprocess.Popen since the older function would fail (more or less) silently if the executed program cannot be found
More info:
http://docs.python.org/library/subprocess.html#subprocess-replacements[YOCTO #2454]
Signed-off-by Robert Yang<[email protected]>--- scripts/contrib/python/generate-manifest-2.7.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)scripts/contrib/python/generate-manifest-2.7.py +1
-0@@ -13,6 +13,7 @@ import os import sys import time +import subprocess VERSION = "2.7.2"@@ -149,7 +150,7 @@ class MakefileMaker: if __name__ == "__main__":
Quoted 10 lines
scripts/contrib/python/generate-manifest-2.7.py class MakefileMaker: · line 156
Quoted 6 lines
scripts/contrib/python/generate-manifest-2.7.py +1
-1if len( sys.argv )> 1: - os.popen( "rm -f ./%s" % sys.argv[1] ) + subprocess.Popen("rm -f ./%s" % sys.argv[1], shell=True, stdout=subprocess.PIPE).stdout outfile = file( sys.argv[1], "w" ) else: outfile = sys.stdoutThis seems like a candidate for subprocess.call, not subprocess.Popen, as the output isn't used. Better yet, just use
os.unlink()and ignore exceptions from it.
Thanks, Chris, I will send the V2 sooner.
// Robert
Quoted 19 lines · Robert Yang
These patches are based on oe-core.
* Testinfo
1) $ bitbake core-image-sato core-image-sato-sdk metatoolchain-sdk with MACHINE=qemuarm,PACKAGE_CLASSES= "package_rpm"2) $ bitbake core-image-sato core-image-sato-sdk metatoolchain-sdk with MACHINE=qemuarm,
PACKAGE_CLASSES= "package_ipk"3) $ bitbake core-image-sato with MACHINE=qemux86,
PACKAGE_CLASSES= "package_deb"4) $ bitbake world with MACHINE=qemux86
// Robert
The following changes since commit
a7532d6b2870a51079c39366def9ae55faeba626:rootfs_rpm.bbclass: save rpmlib rather than remove it (2012-05-29 22:36:39 +0800)
Quoted 8 lines
are available in the git repository at:
git://git.pokylinux.org/poky-contribrobert/meta_subprocess
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=robert/meta_subprocessRobert Yang (4):
meta: replace os.system with subprocess.call
scripts: replace os.system with subprocess.call
meta: replace os.popen with subprocess.Popen
I took the first three of these, thanks.
Quoted 1 line
scripts: replace os.popen with subprocess.Popen
As Chris says, we may as well use an unlink call for this case.
Cheers,
Richard