utils/genrandconfig: add randconfig based --no-toolchains-csv option
PATCH v4 1/1 utils/genrandconfig: add randconfig based --no-toolchains-csv option
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
---
Changes v3 -> v4:
- use fixup_config instead of other retries to filter bad configs
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactor fixup_config control flow
--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)
utils/genrandconfig +206 -9
@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr) + if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines: + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)
@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)
@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break
@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args() # We need the absolute path to use with O=, because the relative
Signature · 6 lines
2.25.1 _______________________________________________ buildroot mailing list [email protected] https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 11 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configs
Well, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g. BR2_ROOTFS_SKELETON_CUSTOM.
[Others may disagree with me of course.]
Regards,
Arnout
Quoted 246 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +199
-9@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr) + if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines: + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file",
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 10 lines
utils/genrandconfig +7
-0+ type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args() # We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 11 lines · Arnout Vandecappelle
Quoted 7 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Quoted 7 lines
Quoted 4 lines
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configsWell, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.
Well it seems to work better since instead of regenerating it tries to fixup the config. I think this gives better statistical coverage of some config options.
Quoted 5 lines
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g.BR2_ROOTFS_SKELETON_CUSTOM.
That looks a bit tricky, although we could probably add it for some cases that we want test coverage for.
Quoted 262 lines
[Others may disagree with me of course.]
Regards,
ArnoutQuoted 255 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +206
-9@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr) + if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines: + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args()
Quoted 1 line
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 1 line
utils/genrandconfig +0
-0# We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 17 lines · James Hilliard
Quoted 16 lines · Arnout Vandecappelle
Quoted 11 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configs
Quoted 6 lines
Quoted 2 lines
Well, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.Well it seems to work better since instead of regenerating it tries to fixup the config. I think this gives better statistical coverage of some config options.
It will create a bias towards the default option for the options you exclude.
See below.
Quoted 33 lines
Quoted 6 lines
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g.BR2_ROOTFS_SKELETON_CUSTOM.That looks a bit tricky, although we could probably add it for some cases that we want test coverage for.
Quoted 23 lines
[Others may disagree with me of course.]
Regards,
ArnoutQuoted 16 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +0
-0@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr)
Quoted 2 lines
Quoted 2 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 319
Quoted 2 lines
utils/genrandconfig +2
-0+ if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines:
Here, the BR2_ROOTFS_SKELETON_CUSTOM_PATH check is basically redundant:
randconfig will never fill something in for the string, so it is always empty.
Quoted 2 lines
Quoted 2 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 321
Quoted 2 lines
utils/genrandconfig +2
-0+ configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n')
Thus, the result is that if a config is generated with
BR2_ROOTFS_SKELETON_CUSTOM=y, it will be removed again and revert to its default.
In this case, there are just two choices so it doesn't make a difference. But
if there are more than two choices, removing one of them creates a bias towards the default option. I wouldn't call this "better statistical coverage". Statistically, taking a completely random sampling and then discarding samples that are somehow not appropriate is considered "better" than fudging inappropriate samples in a certain direction.
Regards,
Arnout
Quoted 235 lines
Quoted 235 lines
Quoted 235 lines
utils/genrandconfig +202
-9+ + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args()
Quoted 1 line
Quoted 1 line
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 1 line
utils/genrandconfig +0
-0# We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 24 lines · Arnout Vandecappelle
Quoted 20 lines · James Hilliard
Quoted 18 lines · Arnout Vandecappelle
Quoted 11 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configsWell, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.
Quoted 6 lines
Quoted 3 lines
Well it seems to work better since instead of regenerating it tries to fixup the config. I think this gives better statistical coverage of some config options.
It will create a bias towards the default option for the options you exclude.
See below.
Yes, I'm aware of that, however this improves coverage as certain options like enabling the kernel build result in a high percentage of invalid configs otherwise which creates a bias towards builds with the kernel build disabled entirely, at least by fixing up invalid kernel configs instead of triggering a full
re-randomization we get more builds with kernels enabled as the invalid configs should no longer create a bias towards kernel builds being disabled for example. Having a bias towards the common defaults seems to be a net improvement here.
Quoted 34 lines
Quoted 34 lines
Quoted 7 lines
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g.BR2_ROOTFS_SKELETON_CUSTOM.That looks a bit tricky, although we could probably add it for some cases that we want test coverage for.
Quoted 23 lines
[Others may disagree with me of course.]
Regards,
ArnoutQuoted 16 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +0
-0@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr)
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 319
Quoted 2 lines
utils/genrandconfig +2
-0+ if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines:Here, the
BR2_ROOTFS_SKELETON_CUSTOM_PATHcheck is basically redundant:
randconfig will never fill something in for the string, so it is always empty.
The fixup_config function is used for configs based on the toolchains csv's as well so this is necessary to prevent accidentally fixing up a valid BR2_ROOTFS_SKELETON_CUSTOM_PATH provided in a toolchain csv file.
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 321
Quoted 3 lines
utils/genrandconfig +2
-0+ configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n')Thus, the result is that if a config is generated with
BR2_ROOTFS_SKELETON_CUSTOM=y, it will be removed again and revert to its default.
Quoted 6 lines
In this case, there are just two choices so it doesn't make a difference. But
if there are more than two choices, removing one of them creates a bias towards the default option. I wouldn't call this "better statistical coverage". Statistically, taking a completely random sampling and then discarding samples that are somehow not appropriate is considered "better" than fudging inappropriate samples in a certain direction.
The main issue is that throwing out invalids creates a heavy bias towards disabling the configs dependencies entirely which is not desirable.
Having a bias towards kernel builds with default options is better than having a very strong bias towards the kernel not being built at all.
Quoted 242 lines
Regards,
ArnoutQuoted 235 lines
Quoted 235 lines
Quoted 235 lines
utils/genrandconfig +202
-9+ + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args()
Quoted 1 line
Quoted 1 line
Quoted 1 line
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 1 line
utils/genrandconfig +0
-0# We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 32 lines · James Hilliard
Quoted 24 lines · Arnout Vandecappelle
Quoted 20 lines · James Hilliard
Quoted 18 lines · Arnout Vandecappelle
Quoted 11 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configsWell, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.Quoted 6 lines
Quoted 3 lines
Well it seems to work better since instead of regenerating it tries to fixup the config. I think this gives better statistical coverage of some config options.
It will create a bias towards the default option for the options you exclude.
See below.
Quoted 9 lines
Yes, I'm aware of that, however this improves coverage as certain options like enabling the kernel build result in a high percentage of invalid configs otherwise which creates a bias towards builds with the kernel build disabled entirely, at least by fixing up invalid kernel configs instead of triggering a full
re-randomization we get more builds with kernels enabled as the invalid configs should no longer create a bias towards kernel builds being disabled for example. Having a bias towards the common defaults seems to be a net improvement here.
Excellent point, indeed. I hadn't thought so far.
Quoted 37 lines
Quoted 31 lines
Quoted 31 lines
Quoted 4 lines
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g.BR2_ROOTFS_SKELETON_CUSTOM.That looks a bit tricky, although we could probably add it for some cases that we want test coverage for.
Quoted 23 lines
[Others may disagree with me of course.]
Regards,
ArnoutQuoted 16 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +0
-0@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr)Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 319
Quoted 2 lines
utils/genrandconfig +2
-0+ if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines:Here, the
BR2_ROOTFS_SKELETON_CUSTOM_PATHcheck is basically redundant:
randconfig will never fill something in for the string, so it is always empty.
Quoted 4 lines
The
fixup_configfunction is used for configs based on the toolchains csv's as well so this is necessary to prevent accidentally fixing up a validBR2_ROOTFS_SKELETON_CUSTOM_PATHprovided in a toolchain csv file.
Oh, right, I didn't think of that either! At the moment the point is moot
because randpackageconfig is used. Oh hang on, I'm wrong again, it actually *would* work specifically for ROOTFS_SKELETON_CUSTOM_PATH, if ROOTFS_SKELETON_CUSTOM is also set in the toolchain config. So I expressed it incorrectly in the commit message.
Unfortunately, I already applied and pushed, so I can't fix the commit message
any more...
Anyway, thanks for the patch!
I think the next steps are:
- move 'make dependencies' inside the loop (not strictly necessary but I think it woudl be an improvement);
- add the toolchain config *after* running randconfig, so we can use full randconfig even if toolchains-csv is set;
- add a mechanism to be able to specify additional defaults together with the toolchain config;
- add options to the toolchain configs to give default values for some string options.
A mechanism is needed for adding defaults because currently we can only force
options in the toolchain config - if it's removed by 'make olddefconfig', is_toolchain_usable will return false. But we also don't want to *always* build a kernel in a toolchain config - and this is particularly true for uboot and barebox, where there's no arch default config to fall back to, so even with randconfig they essentially will never be built.
Regards,
Arnout
Quoted 263 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 321
Quoted 4 lines
utils/genrandconfig +2
-0+ configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n')Thus, the result is that if a config is generated with
BR2_ROOTFS_SKELETON_CUSTOM=y, it will be removed again and revert to its default.Quoted 6 lines
In this case, there are just two choices so it doesn't make a difference. But
if there are more than two choices, removing one of them creates a bias towards the default option. I wouldn't call this "better statistical coverage". Statistically, taking a completely random sampling and then discarding samples that are somehow not appropriate is considered "better" than fudging inappropriate samples in a certain direction.The main issue is that throwing out invalids creates a heavy bias towards disabling the configs dependencies entirely which is not desirable.
Having a bias towards kernel builds with default options is better than having a very strong bias towards the kernel not being built at all.
Quoted 242 lines
Regards,
ArnoutQuoted 235 lines
Quoted 235 lines
Quoted 235 lines
utils/genrandconfig +202
-9+ + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args()
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 1 line
utils/genrandconfig +0
-0# We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot
Quoted 96 lines · Arnout Vandecappelle
Quoted 32 lines · James Hilliard
Quoted 24 lines · Arnout Vandecappelle
Quoted 20 lines · James Hilliard
Quoted 18 lines · Arnout Vandecappelle
Quoted 11 lines · James Hilliard
Currently we only test a limited set of toolchains that are mostly prebuilt, add a flag to allow using randconfig for randomizing additional toolchain settings instead of randpackageconfig.
To avoid invalid configs we need to add additional config validation filtering and fixups.
Signed-off-by James Hilliard <[email protected]>---
Changes v3 -> v4:
- usefixup_configinstead of other retries to filter bad configsWell, I don't like this. It means we have to duplicate the logic for checking
the bad configs in two places: in the mk file and in genrandconfig.Quoted 6 lines
Quoted 3 lines
Well it seems to work better since instead of regenerating it tries to fixup the config. I think this gives better statistical coverage of some config options.
It will create a bias towards the default option for the options you exclude.
See below.Quoted 9 lines
Yes, I'm aware of that, however this improves coverage as certain options like enabling the kernel build result in a high percentage of invalid configs otherwise which creates a bias towards builds with the kernel build disabled entirely, at least by fixing up invalid kernel configs instead of triggering a full
re-randomization we get more builds with kernels enabled as the invalid configs should no longer create a bias towards kernel builds being disabled for example. Having a bias towards the common defaults seems to be a net improvement here.Excellent point, indeed. I hadn't thought so far.
Quoted 37 lines
Quoted 31 lines
Quoted 31 lines
Quoted 4 lines
If the problem is that you sometimes don't end up with a valid config even
after 100 iterations, then I think a better solution is to feed valid values for the string options. That way, we still do a build test of e.g.BR2_ROOTFS_SKELETON_CUSTOM.That looks a bit tricky, although we could probably add it for some cases that we want test coverage for.
Quoted 23 lines
[Others may disagree with me of course.]
Regards,
ArnoutQuoted 16 lines
Changes v2 -> v3:
- properly check exit codes
Changes v1 -> v2:
- refactorfixup_configcontrol flow--- utils/genrandconfig | 215 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 9 deletions(-)utils/genrandconfig +0
-0@@ -315,6 +315,192 @@ def fixup_config(sysinfo, configfile): configlines.remove('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE=""\n') configlines.append('BR2_PACKAGE_HOST_UBOOT_TOOLS_BOOT_SCRIPT_SOURCE="%s"\n' % bootscr)Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 319
Quoted 2 lines
utils/genrandconfig +2
-0+ if 'BR2_ROOTFS_SKELETON_CUSTOM=y\n' in configlines and \ + 'BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n' in configlines:Here, the
BR2_ROOTFS_SKELETON_CUSTOM_PATHcheck is basically redundant:
randconfig will never fill something in for the string, so it is always empty.Quoted 4 lines
The
fixup_configfunction is used for configs based on the toolchains csv's as well so this is necessary to prevent accidentally fixing up a validBR2_ROOTFS_SKELETON_CUSTOM_PATHprovided in a toolchain csv file.Oh, right, I didn't think of that either! At the moment the point is moot
because randpackageconfig is used. Oh hang on, I'm wrong again, it actually *would* work specifically forROOTFS_SKELETON_CUSTOM_PATH, ifROOTFS_SKELETON_CUSTOMis also set in the toolchain config. So I expressed it incorrectly in the commit message.
Quoted 6 lines
Unfortunately, I already applied and pushed, so I can't fix the commit message
any more...Anyway, thanks for the patch!
I think the next steps are:
Well first we need a few fixes in buildroot-test so we can enable the new option:
https://patchwork.ozlabs.org/project/buildroot/patch/[email protected]/
https://patchwork.ozlabs.org/project/buildroot/patch/[email protected]/
I ended up getting rid of docopts since it's unmaintained and makes it tricky to add the --no-toolchains-csv properly.
Quoted 3 lines
- move 'make dependencies' inside the loop (not strictly necessary but I think it woudl be an improvement);
Maybe, although probably need to have failures reported somewhere so that config fixups can be added.
Quoted 10 lines
- add the toolchain config *after* running randconfig, so we can use full randconfig even if toolchains-csv is set;
- add a mechanism to be able to specify additional defaults together with the toolchain config;
- add options to the toolchain configs to give default values for some string options.
Quoted 6 lines
A mechanism is needed for adding defaults because currently we can only force
options in the toolchain config - if it's removed by 'make olddefconfig',is_toolchain_usablewill return false. But we also don't want to *always* build a kernel in a toolchain config - and this is particularly true for uboot and barebox, where there's no arch default config to fall back to, so even with randconfig they essentially will never be built.
Yeah, there's a bit of cleanup needed in general, although we're getting high failure rates just from this improved randomization so maybe best to focus on fixing those issues that this change revealed first so that we can get a better
idea of what combinations need more test coverage.
Quoted 269 lines
Regards,
ArnoutQuoted 263 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
utils/genrandconfig def fixup_config(sysinfo, configfile): · line 321
Quoted 4 lines
utils/genrandconfig +2
-0+ configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM=y\n') + configlines.remove('BR2_ROOTFS_SKELETON_CUSTOM_PATH=""\n')Thus, the result is that if a config is generated with
BR2_ROOTFS_SKELETON_CUSTOM=y, it will be removed again and revert to its default.Quoted 6 lines
In this case, there are just two choices so it doesn't make a difference. But
if there are more than two choices, removing one of them creates a bias towards the default option. I wouldn't call this "better statistical coverage". Statistically, taking a completely random sampling and then discarding samples that are somehow not appropriate is considered "better" than fudging inappropriate samples in a certain direction.The main issue is that throwing out invalids creates a heavy bias towards disabling the configs dependencies entirely which is not desirable.
Having a bias towards kernel builds with default options is better than having a very strong bias towards the kernel not being built at all.
Quoted 242 lines
Regards,
ArnoutQuoted 235 lines
Quoted 235 lines
Quoted 235 lines
utils/genrandconfig +202
-9+ + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_USE_DEFCONFIG=y\n') + configlines.append('BR2_LINUX_KERNEL_USE_ARCH_DEFAULT_CONFIG=y\n') + configlines.remove('BR2_LINUX_KERNEL_DEFCONFIG=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_GIT=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_HG=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_HG=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_SVN=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_SVN=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_REPO_URL=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION=y\n') + configlines.append('BR2_LINUX_KERNEL_LATEST_VERSION=y\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_VERSION_VALUE=""\n') + + if 'BR2_LINUX_KERNEL=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_DTS_SUPPORT=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n' in configlines and \ + 'BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_DTS_SUPPORT=y\n') + configlines.remove('BR2_LINUX_KERNEL_INTREE_DTS_NAME=""\n') + configlines.remove('BR2_LINUX_KERNEL_CUSTOM_DTS_PATH=""\n') + if 'BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_UIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_UIMAGE=y\n') + if 'BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_APPENDED_ZIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_ZIMAGE=y\n') + if 'BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_SIMPLEIMAGE=y\n') + configlines.append('BR2_LINUX_KERNEL_VMLINUX=y\n') + + if 'BR2_LINUX_KERNEL_EXT_AUFS=y\n' in configlines and \ + 'BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n' in configlines: + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS=y\n') + configlines.remove('BR2_LINUX_KERNEL_EXT_AUFS_VERSION=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_PACKAGE_LINUX_BACKPORTS=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_LINUX_BACKPORTS_DEFCONFIG=""\n') + + if 'BR2_KERNEL_HEADERS_VERSION=y\n' in configlines and \ + 'BR2_DEFAULT_KERNEL_VERSION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_VERSION=y\n') + configlines.remove('BR2_DEFAULT_KERNEL_VERSION=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n': + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_GIT=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_REPO_URL=""\n') + + if 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n' in configlines and \ + 'BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n' in configlines: + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL=y\n') + configlines.remove('BR2_KERNEL_HEADERS_CUSTOM_TARBALL_LOCATION=""\n') + + if 'BR2_TARGET_AT91BOOTSTRAP3=y\n' in configlines and \ + 'BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3=y\n') + configlines.remove('BR2_TARGET_AT91BOOTSTRAP3_DEFCONFIG=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_BAREBOX=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_BAREBOX=y\n') + configlines.remove('BR2_TARGET_BAREBOX_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_BAREBOX_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_OPTEE_OS=y\n' in configlines and \ + 'BR2_TARGET_OPTEE_OS_PLATFORM=""\n' in configlines: + configlines.remove('BR2_TARGET_OPTEE_OS=y\n') + configlines.remove('BR2_TARGET_OPTEE_OS_PLATFORM=""\n') + + if 'BR2_TARGET_S500_BOOTLOADER=y\n' in configlines and \ + 'BR2_TARGET_S500_BOOTLOADER_BOARD=""\n' in configlines: + configlines.remove('BR2_TARGET_S500_BOOTLOADER=y\n') + configlines.remove('BR2_TARGET_S500_BOOTLOADER_BOARD=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_CUSTOM_CONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_CUSTOM_CONFIG_FILE=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_KCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_USE_DEFCONFIG=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARD_DEFCONFIG=""\n') + + if 'BR2_TARGET_UBOOT=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n' in configlines and \ + 'BR2_TARGET_UBOOT_BOARDNAME=""\n' in configlines: + configlines.remove('BR2_TARGET_UBOOT=y\n') + configlines.remove('BR2_TARGET_UBOOT_BUILD_SYSTEM_LEGACY=y\n') + configlines.remove('BR2_TARGET_UBOOT_BOARDNAME=""\n') + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_PATH=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PREINSTALLED=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_PATH=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_TOOLCHAIN_EXTERNAL=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n' in configlines and \ + 'BR2_TOOLCHAIN_EXTERNAL_URL=""\n' in configlines: + configlines.remove('BR2_TOOLCHAIN_EXTERNAL=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_CUSTOM=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_DOWNLOAD=y\n') + configlines.remove('BR2_TOOLCHAIN_EXTERNAL_URL=""\n') + if 'BR2_ARCH_HAS_NO_TOOLCHAIN_BUILDROOT=y\n' in configlines: + return False + + if 'BR2_PACKAGE_XVISOR=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n' in configlines and \ + 'BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n' in configlines: + configlines.remove('BR2_PACKAGE_XVISOR_USE_CUSTOM_CONFIG=y\n') + configlines.append('BR2_PACKAGE_XVISOR_USE_DEFCONFIG=y\n') + configlines.remove('BR2_PACKAGE_XVISOR_CUSTOM_CONFIG_FILE=""\n') + with open(configfile, "w+") as configf: configf.writelines(configlines)@@ -331,11 +517,14 @@ def gen_config(args): sysinfo = SystemInfo() - # Select a random toolchain configuration - configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) + if args.toolchains_csv: + # Select a random toolchain configuration + configs = get_toolchain_configs(args.toolchains_csv, args.buildrootdir) - i = randint(0, len(configs) - 1) - toolchainconfig = configs[i] + i = randint(0, len(configs) - 1) + toolchainconfig = configs[i] + else: + toolchainconfig = [] configlines = list(toolchainconfig)@@ -409,7 +598,7 @@ def gen_config(args): bounded_loop -= 1 subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "KCONFIG_PROBABILITY=%d" % randint(1, 20), - "randpackageconfig"]) + "randpackageconfig" if args.toolchains_csv else "randconfig"]) if fixup_config(sysinfo, configfile): break@@ -433,10 +622,18 @@ if __name__ == '__main__': parser.add_argument("--buildrootdir", "-b", help="Buildroot directory (relative to current directory)", type=str, default='.') - parser.add_argument("--toolchains-csv", - help="Path of the toolchain configuration file", - type=str, - default="support/config-fragments/autobuild/toolchain-configs.csv") + + toolchains_csv = parser.add_mutually_exclusive_group(required=False) + toolchains_csv.add_argument("--toolchains-csv", + dest="toolchains_csv", + help="Path of the toolchain configuration file", + type=str) + toolchains_csv.add_argument("--no-toolchains-csv", + dest="toolchains_csv", + help="Generate random toolchain configuration", + action='store_false') + parser.set_defaults(toolchains_csv="support/config-fragments/autobuild/toolchain-configs.csv") + args = parser.parse_args()
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
utils/genrandconfig if __name__ == '__main__': · line 639
Quoted 1 line
utils/genrandconfig +0
-0# We need the absolute path to use with O=, because the relative
_______________________________________________
buildroot mailing list
[email protected]
https://lists.buildroot.org/mailman/listinfo/buildroot