Skip to contentSkip to conversation

utils/genrandconfig: migrate to asyncio subprocess calls

PATCH In review James Hilliard 2 participants 3 messages Apr 13, 2022 – Feb 6, 2023

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 utils/genrandconfig: migrate to asyncio subprocess calls

Since genrandconfig no longer appears to support python2 we can migrate the subprocess calls to use asyncio variants.

This has the advantage of allowing for runners like autobuild-run to integrate directly into genrandconfig by calling the asyncio gen_config using importlib instead of having to run genrandconfig as a subprocess.

Using asyncio is advantageous here as it eliminates the requirement for the runner to deal with blocking subprocess calls(by having to use threading for example).

Also cleanup some unused functions/python2 compatibility shims.

---
Changes v1 -> v2:
- compatibility shims for python 3.4 and newer

---
 utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
 1 file changed, 61 insertions(+), 33 deletions(-)
utils/genrandconfig +61 -33
index 59fe34e58d..aa82a41d5c 100755
@@ -18,23 +18,18 @@
 
 # This script generates a random configuration for testing Buildroot.
 
-import contextlib
+import asyncio
 import csv
 import os
 from random import randint
-import subprocess
 import sys
 from distutils.version import StrictVersion
 import platform
 
-if sys.hexversion >= 0x3000000:
-    import urllib.request as _urllib
+if sys.version_info < (3, 8):
+    from asyncio import coroutine
 else:
-    import urllib2 as _urllib
-
-
-def urlopen_closing(uri):
-    return contextlib.closing(_urllib.urlopen(uri))
+    from types import coroutine
 
 
 class SystemInfo:
@@ -63,6 +58,7 @@ class SystemInfo:
         # --
         return None
 
+    @coroutine
     def has(self, prog):
         """Checks whether a program is available.
         Lazily evaluates missing entries.
@@ -78,11 +74,13 @@ class SystemInfo:
         have_it = self.find_prog(prog)
         # java[c] needs special care
         if have_it and prog in ('java', 'javac'):
-            with open(os.devnull, "w") as devnull:
-                if subprocess.call("%s -version | grep gcj" % prog,
-                                   shell=True,
-                                   stdout=devnull, stderr=devnull) != 1:
-                    have_it = False
+            proc = yield from asyncio.create_subprocess_shell(
+                "%s -version | grep gcj" % prog,
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.DEVNULL)
+            ret = yield from proc.wait()
+            if ret != 1:
+                have_it = False
         # --
         self.progs[prog] = have_it
         return have_it
@@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
     return configs
 
 
+@coroutine
 def is_toolchain_usable(configfile, config):
     """Check if the toolchain is actually usable."""
 
@@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
            'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
            'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
            'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
-            ldd_version_output = subprocess.check_output(['ldd', '--version'])
-            glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+            proc = yield from asyncio.create_subprocess_exec(
+                'ldd', '--version', stdout=asyncio.subprocess.PIPE)
+            ldd_version_output, _ = yield from proc.communicate()
+            if proc.returncode:
+                return False
+            glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')
             if StrictVersion('2.14') > StrictVersion(glibc_version):
                 print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
                 return False
@@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
     return True
 
 
+@coroutine
 def fixup_config(sysinfo, configfile):
     """Finalize the configuration and reject any problematic combinations
 
@@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
 
     BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
 
-    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+    has_java = yield from sysinfo.has("java")
+    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
         return False
     # The ctng toolchain is affected by PR58854
     if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
@@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
     return True
 
 
+@coroutine
 def gen_config(args):
     """Generate a new random configuration
 
@@ -565,7 +571,8 @@ def gen_config(args):
 
     # Randomly enable BR2_REPRODUCIBLE 10% of times
     # also enable tar filesystem images for testing
-    if sysinfo.has("diffoscope") and randint(0, 10) == 0:
+    has_diffoscope = yield from sysinfo.has("diffoscope")
+    if has_diffoscope and randint(0, 10) == 0:
         configlines.append("BR2_REPRODUCIBLE=y\n")
         configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
 
@@ -579,10 +586,14 @@ def gen_config(args):
     with open(configfile, "w+") as configf:
         configf.writelines(configlines)
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    if not is_toolchain_usable(configfile, toolchainconfig):
+    toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
+    if not toolchain_usable:
         return 2
 
     # Now, generate the random selection of packages, and fixup
@@ -596,21 +607,34 @@ def gen_config(args):
                   file=sys.stderr)
             return 1
         bounded_loop -= 1
-        subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                               "KCONFIG_PROBABILITY=%d" % randint(1, 20),
-                               "randpackageconfig" if args.toolchains_csv else "randconfig"])
-
-        if fixup_config(sysinfo, configfile):
+        proc = yield from asyncio.create_subprocess_exec(
+            "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
+            "KCONFIG_PROBABILITY=%d" % randint(1, 20),
+            "randpackageconfig" if args.toolchains_csv else "randconfig")
+        ret = yield from proc.wait()
+        if ret:
+            return ret
+
+        ret = yield from fixup_config(sysinfo, configfile)
+        if ret:
             break
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "savedefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                            "dependencies"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
+    ret = yield from proc.wait()
+    return ret
 
 
 if __name__ == '__main__':
@@ -642,7 +666,11 @@ if __name__ == '__main__':
     args.outputdir = os.path.abspath(args.outputdir)
 
     try:
-        ret = gen_config(args)
+        if sys.version_info < (3, 7):
+            loop = asyncio.get_event_loop()
+            ret = loop.run_until_complete(gen_config(args))
+        else:
+            ret = asyncio.run(gen_config(args))
     except Exception as e:
         print(str(e), file=sys.stderr)
         parser.exit(1)
Signature · 6 lines
2.25.1

_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
#1 lore raw

Hi James,

Quoted 8 lines · James Hilliard

Since genrandconfig no longer appears to support python2 we can migrate the subprocess calls to use asyncio variants.

This has the advantage of allowing for runners like autobuild-run to integrate directly into genrandconfig by calling the asyncio gen_config using importlib instead of having to run genrandconfig as a subprocess.

Quoted 7 lines

Using asyncio is advantageous here as it eliminates the requirement for the runner to deal with blocking subprocess calls(by having to use threading for example).

Also cleanup some unused functions/python2 compatibility shims.

I finally applied to master. Needed to resolve some conflicts, but otherwise
no changes. I still do have a few comments though.

Quoted 40 lines

---
Changes v1 -> v2:
- compatibility shims for python 3.4 and newer

---
  utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
  1 file changed, 61 insertions(+), 33 deletions(-)
utils/genrandconfig +4 -9
index 59fe34e58d..aa82a41d5c 100755
@@ -18,23 +18,18 @@
 
  # This script generates a random configuration for testing Buildroot.
 
-import contextlib
+import asyncio
  import csv
  import os
  from random import randint
-import subprocess
  import sys
  from distutils.version import StrictVersion
  import platform
 
-if sys.hexversion >= 0x3000000:
-    import urllib.request as _urllib
+if sys.version_info < (3, 8):
+    from asyncio import coroutine
  else:
-    import urllib2 as _urllib
-
-
-def urlopen_closing(uri):
-    return contextlib.closing(_urllib.urlopen(uri))
+    from types import coroutine
 
 
  class SystemInfo:

utils/genrandconfig class SystemInfo: · line 62

Quoted 5 lines
utils/genrandconfig +1 -0
@@ -63,6 +58,7 @@ class SystemInfo:
          # --
          return None
 
+    @coroutine
      def has(self, prog):

Isn't the usual way to use

async def has(self, prog):

? Or does that mean something different?

utils/genrandconfig class SystemInfo: · line 78

Quoted 13 lines
utils/genrandconfig +2 -5
          """Checks whether a program is available.
          Lazily evaluates missing entries.
@@ -78,11 +74,13 @@ class SystemInfo:
          have_it = self.find_prog(prog)
          # java[c] needs special care
          if have_it and prog in ('java', 'javac'):
-            with open(os.devnull, "w") as devnull:
-                if subprocess.call("%s -version | grep gcj" % prog,
-                                   shell=True,
-                                   stdout=devnull, stderr=devnull) != 1:
-                    have_it = False
+            proc = yield from asyncio.create_subprocess_shell(
+                "%s -version | grep gcj" % prog,

Perhaps we should also start using f-strings. That bumps the requirement to
python 3.6 but I don't think it's unreasonable for genrandconfig.

Quoted 18 lines
utils/genrandconfig +6 -0
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.DEVNULL)
+            ret = yield from proc.wait()
+            if ret != 1:
+                have_it = False
          # --
          self.progs[prog] = have_it
          return have_it
@@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
      return configs
 
 
+@coroutine
  def is_toolchain_usable(configfile, config):
      """Check if the toolchain is actually usable."""
 
@@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \

utils/genrandconfig def is_toolchain_usable(configfile, config): · line 186

Quoted 10 lines
utils/genrandconfig +6 -2
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
-            ldd_version_output = subprocess.check_output(['ldd', '--version'])
-            glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+            proc = yield from asyncio.create_subprocess_exec(
+                'ldd', '--version', stdout=asyncio.subprocess.PIPE)
+            ldd_version_output, _ = yield from proc.communicate()
+            if proc.returncode:
+                return False
+            glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')

The decode was already added in commit
12e4f7c5c43fb7b4db6d6548b9dbabb9c1b5f875 so I didn't modify this line.

Quoted 60 lines
utils/genrandconfig +13 -5
              if StrictVersion('2.14') > StrictVersion(glibc_version):
                  print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
                  return False
@@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
      return True
 
 
+@coroutine
  def fixup_config(sysinfo, configfile):
      """Finalize the configuration and reject any problematic combinations
 
@@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
 
      BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
 
-    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+    has_java = yield from sysinfo.has("java")
+    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
          return False
      # The ctng toolchain is affected by PR58854
      if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
@@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
      return True
 
 
+@coroutine
  def gen_config(args):
      """Generate a new random configuration
 
@@ -565,7 +571,8 @@ def gen_config(args):
 
      # Randomly enable BR2_REPRODUCIBLE 10% of times
      # also enable tar filesystem images for testing
-    if sysinfo.has("diffoscope") and randint(0, 10) == 0:
+    has_diffoscope = yield from sysinfo.has("diffoscope")
+    if has_diffoscope and randint(0, 10) == 0:
          configlines.append("BR2_REPRODUCIBLE=y\n")
          configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
 
@@ -579,10 +586,14 @@ def gen_config(args):
      with open(configfile, "w+") as configf:
          configf.writelines(configlines)
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    if not is_toolchain_usable(configfile, toolchainconfig):
+    toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
+    if not toolchain_usable:
          return 2
 
      # Now, generate the random selection of packages, and fixup
@@ -596,21 +607,34 @@ def gen_config(args):
                    file=sys.stderr)
              return 1

utils/genrandconfig def gen_config(args): · line 613

Quoted 10 lines
utils/genrandconfig +4 -5
          bounded_loop -= 1
-        subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                               "KCONFIG_PROBABILITY=%d" % randint(1, 20),
-                               "randpackageconfig" if args.toolchains_csv else "randconfig"])
-
-        if fixup_config(sysinfo, configfile):
+        proc = yield from asyncio.create_subprocess_exec(
+            "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
+            "KCONFIG_PROBABILITY=%d" % randint(1, 20),
+            "randpackageconfig" if args.toolchains_csv else "randconfig")

This was refactored a bit, can you check if I fixed it up correctly?

Regards,
Arnout

Quoted 36 lines
utils/genrandconfig +20 -6
+        ret = yield from proc.wait()
+        if ret:
+            return ret
+
+        ret = yield from fixup_config(sysinfo, configfile)
+        if ret:
              break
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "savedefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                            "dependencies"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
+    ret = yield from proc.wait()
+    return ret
 
 
  if __name__ == '__main__':
@@ -642,7 +666,11 @@ if __name__ == '__main__':
      args.outputdir = os.path.abspath(args.outputdir)
 

utils/genrandconfig if __name__ == '__main__': · line 676

Quoted 10 lines
utils/genrandconfig +5 -1
      try:
-        ret = gen_config(args)
+        if sys.version_info < (3, 7):
+            loop = asyncio.get_event_loop()
+            ret = loop.run_until_complete(gen_config(args))
+        else:
+            ret = asyncio.run(gen_config(args))
      except Exception as e:
          print(str(e), file=sys.stderr)
          parser.exit(1)

_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot

#2 lore raw
Quoted 66 lines · Arnout Vandecappelle

Hi James,

Quoted 8 lines · James Hilliard

Since genrandconfig no longer appears to support python2 we can migrate the subprocess calls to use asyncio variants.

This has the advantage of allowing for runners like autobuild-run to integrate directly into genrandconfig by calling the asyncio gen_config using importlib instead of having to run genrandconfig as a subprocess.

Quoted 7 lines

Using asyncio is advantageous here as it eliminates the requirement for the runner to deal with blocking subprocess calls(by having to use threading for example).

Also cleanup some unused functions/python2 compatibility shims.

I finally applied to master. Needed to resolve some conflicts, but otherwise
no changes. I still do have a few comments though.

Quoted 43 lines

---
Changes v1 -> v2:
- compatibility shims for python 3.4 and newer

---
  utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
  1 file changed, 61 insertions(+), 33 deletions(-)
utils/genrandconfig +4 -9
index 59fe34e58d..aa82a41d5c 100755
@@ -18,23 +18,18 @@
 
  # This script generates a random configuration for testing Buildroot.
 
-import contextlib
+import asyncio
  import csv
  import os
  from random import randint
-import subprocess
  import sys
  from distutils.version import StrictVersion
  import platform
 
-if sys.hexversion >= 0x3000000:
-    import urllib.request as _urllib
+if sys.version_info < (3, 8):
+    from asyncio import coroutine
  else:
-    import urllib2 as _urllib
-
-
-def urlopen_closing(uri):
-    return contextlib.closing(_urllib.urlopen(uri))
+    from types import coroutine
 
 
  class SystemInfo:
@@ -63,6 +58,7 @@ class SystemInfo:
          # --
          return None
 
Quoted 8 lines

utils/genrandconfig class SystemInfo: · line 62

Quoted 2 lines
utils/genrandconfig +1 -0
+    @coroutine
      def has(self, prog):

Isn't the usual way to use

async def has(self, prog):

? Or does that mean something different?

Yeah, the weird style here was for python 3.4 compat, but if 3.6 is fine this should work:
https://patchwork.ozlabs.org/project/buildroot/patch/[email protected]/

Quoted 18 lines
Quoted 5 lines
utils/genrandconfig +0 -0
          """Checks whether a program is available.
          Lazily evaluates missing entries.

utils/genrandconfig class SystemInfo: · line 78

Quoted 10 lines
utils/genrandconfig +2 -5
@@ -78,11 +74,13 @@ class SystemInfo:
          have_it = self.find_prog(prog)
          # java[c] needs special care
          if have_it and prog in ('java', 'javac'):
-            with open(os.devnull, "w") as devnull:
-                if subprocess.call("%s -version | grep gcj" % prog,
-                                   shell=True,
-                                   stdout=devnull, stderr=devnull) != 1:
-                    have_it = False
+            proc = yield from asyncio.create_subprocess_shell(
+                "%s -version | grep gcj" % prog,

Perhaps we should also start using f-strings. That bumps the requirement to
python 3.6 but I don't think it's unreasonable for genrandconfig.

Yeah, older than 3.6 starts getting pretty annoying.

Quoted 89 lines
Quoted 19 lines
utils/genrandconfig +6 -0
+                stdout=asyncio.subprocess.DEVNULL,
+                stderr=asyncio.subprocess.DEVNULL)
+            ret = yield from proc.wait()
+            if ret != 1:
+                have_it = False
          # --
          self.progs[prog] = have_it
          return have_it
@@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
      return configs
 
 
+@coroutine
  def is_toolchain_usable(configfile, config):
      """Check if the toolchain is actually usable."""
 
@@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \

utils/genrandconfig def is_toolchain_usable(configfile, config): · line 186

Quoted 10 lines
utils/genrandconfig +6 -2
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
             'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
-            ldd_version_output = subprocess.check_output(['ldd', '--version'])
-            glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+            proc = yield from asyncio.create_subprocess_exec(
+                'ldd', '--version', stdout=asyncio.subprocess.PIPE)
+            ldd_version_output, _ = yield from proc.communicate()
+            if proc.returncode:
+                return False
+            glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')

The decode was already added in commit
12e4f7c5c43fb7b4db6d6548b9dbabb9c1b5f875 so I didn't modify this line.

Quoted 56 lines
utils/genrandconfig +13 -5
              if StrictVersion('2.14') > StrictVersion(glibc_version):
                  print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
                  return False
@@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
      return True
 
 
+@coroutine
  def fixup_config(sysinfo, configfile):
      """Finalize the configuration and reject any problematic combinations
 
@@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
 
      BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
 
-    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+    has_java = yield from sysinfo.has("java")
+    if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
          return False
      # The ctng toolchain is affected by PR58854
      if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
@@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
      return True
 
 
+@coroutine
  def gen_config(args):
      """Generate a new random configuration
 
@@ -565,7 +571,8 @@ def gen_config(args):
 
      # Randomly enable BR2_REPRODUCIBLE 10% of times
      # also enable tar filesystem images for testing
-    if sysinfo.has("diffoscope") and randint(0, 10) == 0:
+    has_diffoscope = yield from sysinfo.has("diffoscope")
+    if has_diffoscope and randint(0, 10) == 0:
          configlines.append("BR2_REPRODUCIBLE=y\n")
          configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
 
@@ -579,10 +586,14 @@ def gen_config(args):
      with open(configfile, "w+") as configf:
          configf.writelines(configlines)
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    if not is_toolchain_usable(configfile, toolchainconfig):
+    toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
+    if not toolchain_usable:
          return 2
 
Quoted 16 lines
Quoted 4 lines
utils/genrandconfig +0 -0
      # Now, generate the random selection of packages, and fixup
@@ -596,21 +607,34 @@ def gen_config(args):
                    file=sys.stderr)
              return 1

utils/genrandconfig def gen_config(args): · line 613

Quoted 10 lines
utils/genrandconfig +4 -5
          bounded_loop -= 1
-        subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                               "KCONFIG_PROBABILITY=%d" % randint(1, 20),
-                               "randpackageconfig" if args.toolchains_csv else "randconfig"])
-
-        if fixup_config(sysinfo, configfile):
+        proc = yield from asyncio.create_subprocess_exec(
+            "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
+            "KCONFIG_PROBABILITY=%d" % randint(1, 20),
+            "randpackageconfig" if args.toolchains_csv else "randconfig")

This was refactored a bit, can you check if I fixed it up correctly?

Yeah, looks good from what I can tell.

Quoted 40 lines

Regards,
Arnout

Quoted 36 lines
utils/genrandconfig +20 -6
+        ret = yield from proc.wait()
+        if ret:
+            return ret
+
+        ret = yield from fixup_config(sysinfo, configfile)
+        if ret:
              break
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "olddefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                           "savedefconfig"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
+    ret = yield from proc.wait()
+    if ret:
+        return ret
 
-    return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
-                            "dependencies"])
+    proc = yield from asyncio.create_subprocess_exec(
+        "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
+    ret = yield from proc.wait()
+    return ret
 
 
  if __name__ == '__main__':
@@ -642,7 +666,11 @@ if __name__ == '__main__':
      args.outputdir = os.path.abspath(args.outputdir)
 
Quoted 10 lines

utils/genrandconfig if __name__ == '__main__': · line 676

Quoted 10 lines
utils/genrandconfig +5 -1
      try:
-        ret = gen_config(args)
+        if sys.version_info < (3, 7):
+            loop = asyncio.get_event_loop()
+            ret = loop.run_until_complete(gen_config(args))
+        else:
+            ret = asyncio.run(gen_config(args))
      except Exception as e:
          print(str(e), file=sys.stderr)
          parser.exit(1)

_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot