#11552 - wic: avoid unnecessary dependencies
v2 cover #11552 - wic: avoid unnecessary dependencies
Hi,
This patchset gets rid of building wic-tools for every image when IMAGE_FSTYPES contains 'wic'. Instead of using dependencies from wic-tools native sysroot wic expects image native sysroot to contain them. This should be done by setting WKS_FILE_DEPENDS variable in configuration files where WKS_FILE is set.
wic-tools can be used by wic when its run manually by pointing to the native sysroot using --native-sysroot option.
The patchset also contains fixes for the isoimage plugin and wic test suite breakages that were coused by the new changes.
Changes in v2:
- Fixed 2 more test cases
- Fixed default WKS_FILE_DEPENDS value (thanks to Patrick Ohly)
- Rebased on top of recent poky master
The following changes since commit f519ac010a7f7e018a83d85b6dc7778904b9a703:
oe-selftest: wic: fix test_quemu (2017-07-12 14:33:02 +0300)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib ed/wic/wip
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wic/wip
Ed Bartosh (8):
wic: get rid of using wic-tools
wic: isoimage: do not remove temp directory
wic: build wic-tools only if wic is run manually
wic: isoimage-isohybrid: check result of glob()
wic-tools: don't create wic-tools.env
image_types_wic: set default WKS_FILE_DEPENDS
wic: isoimage-isohybrid: use grub-efi from deploy dir
selftest: fixed 5 wic test cases
meta/classes/image_types_wic.bbclass | 11 ++- meta/lib/oeqa/selftest/cases/wic.py | 23 +++++- meta/recipes-core/meta/wic-tools.bb | 10 +-- scripts/lib/wic/misc.py | 9 +-- scripts/lib/wic/plugins/source/bootimg-pcbios.py | 5 +- .../lib/wic/plugins/source/isoimage-isohybrid.py | 88 ++++++++-------------- scripts/wic | 14 ++-- 7 files changed, 73 insertions(+), 87 deletions(-)
--
Regards,
Ed
PATCH v2 1/8 wic: get rid of using wic-tools
At the moment, when building images with IMAGE_FSTYPES=wic one ends up depending on wic-tools and thus syslinux and grub-efi even when not using those at all. Ideally, building an image with wic should only build the tools and components really needed.
The problem is that "wic-tools" is needed also for the manual invocations of wic, in which case everything that might be needed has to be built in advance.
Replaced dependency on wic-tools with dependency to a much shorter set of tools that wic uses almost for any image: 'parted', 'gptfdisk', 'dosfstools' and 'mtools'.
[YOCTO #11552]
--- meta/classes/image_types_wic.bbclass | 2 +- scripts/lib/wic/misc.py | 9 +++------ scripts/lib/wic/plugins/source/bootimg-pcbios.py | 5 ++--- scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 4 ++-- scripts/wic | 6 ++++-- 5 files changed, 12 insertions(+), 14 deletions(-)
meta/classes/image_types_wic.bbclass +1 -1
@@ -39,7 +39,7 @@ IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES" USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}" WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}" do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}" -do_image_wic[depends] += "wic-tools:do_populate_sysroot" +do_image_wic[depends] += "${@' '.join('%s-native:do_populate_sysroot' % r for r in ('parted', 'gptfdisk', 'dosfstools', 'mtools'))}" WKS_FILE_DEPENDS ??= '' DEPENDS += "${@ '${WKS_FILE_DEPENDS}' if d.getVar('USING_WIC') else '' }"
scripts/lib/wic/misc.py +3 -6
@@ -139,14 +139,11 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): if pseudo: cmd_and_args = pseudo + cmd_and_args - wtools_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", "wic-tools") + native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin" % \ + (native_sysroot, native_sysroot, native_sysroot) - native_paths = \ - "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/sbin:%s/usr/sbin:%s/usr/bin" % \ - (wtools_sysroot, wtools_sysroot, wtools_sysroot, - native_sysroot, native_sysroot, native_sysroot) native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ - (native_paths, cmd_and_args) + (native_paths, cmd_and_args) logger.debug("exec_native_cmd: %s", native_cmd_and_args) # If the command isn't in the native sysroot say we failed.
scripts/lib/wic/plugins/source/bootimg-pcbios.py +2 -3
@@ -45,10 +45,9 @@ class BootimgPcbiosPlugin(SourcePlugin): @classmethod def _get_bootimg_dir(cls, bootimg_dir, dirname): """ - Check if dirname exists in default bootimg_dir or - in wic-tools STAGING_DIR. + Check if dirname exists in default bootimg_dir or in STAGING_DIR. """ - for result in (bootimg_dir, get_bitbake_var("STAGING_DATADIR", "wic-tools")): + for result in (bootimg_dir, get_bitbake_var("STAGING_DATADIR")): if os.path.exists("%s/%s" % (result, dirname)): return result
scripts/lib/wic/plugins/source/isoimage-isohybrid.py +2 -2
@@ -342,7 +342,7 @@ class IsoImagePlugin(SourcePlugin): if not os.path.isfile("%s/EFI/BOOT/%s" \ % (bootimg_dir, grub_image)): - grub_path = get_bitbake_var("STAGING_LIBDIR", "wic-tools") + grub_path = get_bitbake_var("STAGING_LIBDIR") if not grub_path: raise WicError("Couldn't find STAGING_LIBDIR, exiting.")
@@ -411,7 +411,7 @@ class IsoImagePlugin(SourcePlugin): exec_cmd(chmod_cmd) # Prepare files for legacy boot - syslinux_dir = get_bitbake_var("STAGING_DATADIR", "wic-tools") + syslinux_dir = get_bitbake_var("STAGING_DATADIR") if not syslinux_dir: raise WicError("Couldn't find STAGING_DATADIR, exiting.")
scripts/wic +4 -2
@@ -154,8 +154,10 @@ def wic_create_subcommand(options, usage_str): rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", options.image_name) kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE", options.image_name) bootimg_dir = get_bitbake_var("STAGING_DATADIR", options.image_name) - native_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", - options.image_name) #, cache=False) + + native_sysroot = options.native_sysroot + if options.vars_dir and not native_sysroot: + native_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", options.image_name) else: if options.build_rootfs: raise WicError("Image name is not specified, exiting. "
Signature · 1 line
2.1.4
PATCH v2 2/8 wic: isoimage: do not remove temp directory
In reply to Ed BartoshRemoved isodir subrdirectory instead of removing temporary working directory as working directory can contain copy of rootfs partition and shouldn't be removed by any plugin.
--- scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
scripts/lib/wic/plugins/source/isoimage-isohybrid.py +2 -2
@@ -204,8 +204,8 @@ class IsoImagePlugin(SourcePlugin): """ isodir = "%s/ISO/" % cr_workdir - if os.path.exists(cr_workdir): - shutil.rmtree(cr_workdir) + if os.path.exists(isodir): + shutil.rmtree(isodir) install_cmd = "install -d %s " % isodir exec_cmd(install_cmd)
Signature · 1 line
2.1.4
PATCH v2 3/8 wic: build wic-tools only if wic is run manually
In reply to Ed BartoshWhen wic can't find native sysroot it tries to build wic-tools However, it's not possible when wic is run from bitbake. Moreover, it's not even feasible anymore as wic-tools should be used only when wic is run manually.
Checked if wic is run manually before building wic-tools.
--- scripts/wic | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
scripts/wic +4 -4
@@ -164,15 +164,15 @@ def wic_create_subcommand(options, usage_str): "(Use -e/--image-name to specify it)") native_sysroot = options.native_sysroot - if not native_sysroot or not os.path.isdir(native_sysroot): + if not options.vars_dir and (not native_sysroot or not os.path.isdir(native_sysroot)): logger.info("Building wic-tools...\n") if bitbake_main(BitBakeConfigParameters("bitbake wic-tools".split()), cookerdata.CookerConfiguration()): raise WicError("bitbake wic-tools failed") native_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", "wic-tools") - if not native_sysroot: - raise WicError("Unable to find the location of the native " - "tools sysroot to use") + + if not native_sysroot: + raise WicError("Unable to find the location of the native tools sysroot") wks_file = options.wks_file
Signature · 1 line
2.1.4
PATCH v2 4/8 wic: isoimage-isohybrid: check result of glob()
In reply to Ed Bartoshisoimage-isohybrid plugin uses result of glob call to get path to initrd image. When glob returns empty list the plugin crashes with IndexError.
Checking if result of glob call is not empty should fix the breakage.
--- scripts/lib/wic/plugins/source/isoimage-isohybrid.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
scripts/lib/wic/plugins/source/isoimage-isohybrid.py +5 -2
@@ -164,9 +164,12 @@ class IsoImagePlugin(SourcePlugin): machine = os.path.basename(initrd_dir) - initrd = glob.glob('%s/%s*%s.%s' % (initrd_dir, image_name, machine, image_type))[0] + pattern = '%s/%s*%s.%s' % (initrd_dir, image_name, machine, image_type) + files = glob.glob(pattern) + if files: + initrd = files[0] - if not os.path.exists(initrd): + if not initrd or not os.path.exists(initrd): # Create initrd from rootfs directory initrd = "%s/initrd.cpio.gz" % cr_workdir initrd_dir = "%s/INITRD" % cr_workdir
Signature · 1 line
2.1.4
PATCH v2 5/8 wic-tools: don't create wic-tools.env
In reply to Ed Bartoshwic-tools.env was uses only when wic is run from bitbake. As wic doesn't use wic-tools anymore in this mode there is no need for this file.
--- meta/recipes-core/meta/wic-tools.bb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-)
meta/recipes-core/meta/wic-tools.bb +1 -9
@@ -11,6 +11,7 @@ DEPENDS_append_x86 = " syslinux grub-efi systemd-boot" DEPENDS_append_x86-64 = " syslinux grub-efi systemd-boot" INHIBIT_DEFAULT_DEPS = "1" + inherit nopackages # The sysroot of wic-tools is needed for wic, but if rm_work is enabled, it will
@@ -19,14 +20,5 @@ RM_WORK_EXCLUDE += "${PN}" python do_build_sysroot () { bb.build.exec_func("extend_recipe_sysroot", d) - - # Write environment variables used by wic - # to tmp/sysroots/<machine>/imgdata/wictools.env - outdir = os.path.join(d.getVar('STAGING_DIR'), d.getVar('MACHINE'), 'imgdata') - bb.utils.mkdirhier(outdir) - with open(os.path.join(outdir, "wic-tools.env"), 'w') as envf: - for var in ('RECIPE_SYSROOT_NATIVE', 'STAGING_DATADIR', 'STAGING_LIBDIR'): - envf.write('%s="%s"\n' % (var, d.getVar(var).strip())) - } addtask do_build_sysroot after do_prepare_recipe_sysroot before do_build
Signature · 1 line
2.1.4
PATCH v2 6/8 image_types_wic: set default WKS_FILE_DEPENDS
In reply to Ed BartoshThe default set of dependencies used via wic-tools is not used anymore as wic images don't depend on wic-tools.
Set of packages required to produce wic image depends on the content of the .wks file, so WKS_FILE_DEPENDS variable should be used to provide correct list of dependencies when WKS_FILE variable is set.
As WKS_FILE_DEPENDS is not used in many existing recipes yet we need to provide default value for it to ensure that removal of wic-tool dependency doesn't cause image build failures.
Initialized WKS_FILE_DEPENDS with the set of dependencies previously brought by wic-tool. This is done to provide compatibility and to avoid breakages of existing image recipes that rely on current set of dependencies.
Note: This is a temporary solution for transition period. After some time the list will be either reduced or removed. Recommended solution is to use WKS_FILE_DEPENDS in image recipes together with WKS_FILE to specify dependencies.
--- meta/classes/image_types_wic.bbclass | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
meta/classes/image_types_wic.bbclass +8 -1
@@ -40,7 +40,14 @@ USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}" do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}" do_image_wic[depends] += "${@' '.join('%s-native:do_populate_sysroot' % r for r in ('parted', 'gptfdisk', 'dosfstools', 'mtools'))}" -WKS_FILE_DEPENDS ??= '' + +WKS_FILE_DEPENDS_DEFAULT = "syslinux-native bmap-tools-native cdrtools-native btrfs-tools-native squashfs-tools-native" +WKS_FILE_DEPENDS_BOOTLOADERS = "" +WKS_FILE_DEPENDS_BOOTLOADERS_x86 = "syslinux grub-efi systemd-boot" +WKS_FILE_DEPENDS_BOOTLOADERS_x86-64 = "syslinux grub-efi systemd-boot" + +WKS_FILE_DEPENDS ??= "${WKS_FILE_DEPENDS_DEFAULT} ${WKS_FILE_DEPENDS_BOOTLOADERS}" + DEPENDS += "${@ '${WKS_FILE_DEPENDS}' if d.getVar('USING_WIC') else '' }" python do_write_wks_template () {
Signature · 1 line
2.1.4
PATCH v2 7/8 wic: isoimage-isohybrid: use grub-efi from deploy dir
In reply to Ed Bartoshisoimage-isohybrid plugin tries to build grub-efi in its working directory if it can't find efi binary. Wic should avoid doing anything in working directories of other recipes. It should use artifacts from the image deployment directory instead.
--- .../lib/wic/plugins/source/isoimage-isohybrid.py | 75 +++++++--------------- 1 file changed, 23 insertions(+), 52 deletions(-)
scripts/lib/wic/plugins/source/isoimage-isohybrid.py +23 -52
@@ -95,7 +95,7 @@ class IsoImagePlugin(SourcePlugin): cfg.write(syslinux_conf) @classmethod - def do_configure_grubefi(cls, part, creator, cr_workdir): + def do_configure_grubefi(cls, part, creator, target_dir): """ Create loader-specific (grub-efi) config """
@@ -109,7 +109,7 @@ class IsoImagePlugin(SourcePlugin): raise WicError("configfile is specified " "but failed to get it from %s", configfile) else: - splash = os.path.join(cr_workdir, "EFI/boot/splash.jpg") + splash = os.path.join(target_dir, "splash.jpg") if os.path.exists(splash): splashline = "menu background splash.jpg" else:
@@ -137,9 +137,10 @@ class IsoImagePlugin(SourcePlugin): if splashline: grubefi_conf += "%s\n" % splashline - logger.debug("Writing grubefi config %s/EFI/BOOT/grub.cfg", cr_workdir) + cfg_path = os.path.join(target_dir, "grub.cfg") + logger.debug("Writing grubefi config %s", cfg_path) - with open("%s/EFI/BOOT/grub.cfg" % cr_workdir, "w") as cfg: + with open(cfg_path, "w") as cfg: cfg.write(grubefi_conf) @staticmethod
@@ -313,20 +314,13 @@ class IsoImagePlugin(SourcePlugin): #Create bootloader for efi boot try: - if source_params['loader'] == 'grub-efi': - # Builds grub.cfg if ISODIR didn't exist or - # didn't contains grub.cfg - bootimg_dir = img_iso_dir - if not os.path.exists("%s/EFI/BOOT" % bootimg_dir): - bootimg_dir = "%s/bootimg" % cr_workdir - if os.path.exists(bootimg_dir): - shutil.rmtree(bootimg_dir) - install_cmd = "install -d %s/EFI/BOOT" % bootimg_dir - exec_cmd(install_cmd) - - if not os.path.isfile("%s/EFI/BOOT/boot.cfg" % bootimg_dir): - cls.do_configure_grubefi(part, creator, bootimg_dir) + target_dir = "%s/EFI/BOOT" % isodir + if os.path.exists(target_dir): + shutil.rmtree(target_dir) + + os.makedirs(target_dir) + if source_params['loader'] == 'grub-efi': # Builds bootx64.efi/bootia32.efi if ISODIR didn't exist or # didn't contains it target_arch = get_bitbake_var("TARGET_SYS")
@@ -334,37 +328,23 @@ class IsoImagePlugin(SourcePlugin): raise WicError("Coludn't find target architecture") if re.match("x86_64", target_arch): - grub_target = 'x86_64-efi' - grub_image = "bootx64.efi" + grub_image = "grub-efi-bootx64.efi" elif re.match('i.86', target_arch): - grub_target = 'i386-efi' - grub_image = "bootia32.efi" + grub_image = "grub-efi-bootia32.efi" else: raise WicError("grub-efi is incompatible with target %s" % target_arch) - if not os.path.isfile("%s/EFI/BOOT/%s" \ - % (bootimg_dir, grub_image)): - grub_path = get_bitbake_var("STAGING_LIBDIR") - if not grub_path: - raise WicError("Couldn't find STAGING_LIBDIR, exiting.") - - grub_core = "%s/grub/%s" % (grub_path, grub_target) - if not os.path.exists(grub_core): - raise WicError("Please build grub-efi first") - - grub_cmd = "grub-mkimage -p '/EFI/BOOT' " - grub_cmd += "-d %s " % grub_core - grub_cmd += "-O %s -o %s/EFI/BOOT/%s " \ - % (grub_target, bootimg_dir, grub_image) - grub_cmd += "part_gpt part_msdos ntfs ntfscomp fat ext2 " - grub_cmd += "normal chain boot configfile linux multiboot " - grub_cmd += "search efi_gop efi_uga font gfxterm gfxmenu " - grub_cmd += "terminal minicmd test iorw loadenv echo help " - grub_cmd += "reboot serial terminfo iso9660 loopback tar " - grub_cmd += "memdisk ls search_fs_uuid udf btrfs xfs lvm " - grub_cmd += "reiserfs ata " - exec_native_cmd(grub_cmd, native_sysroot) + grub_target = os.path.join(target_dir, grub_image) + if not os.path.isfile(grub_target): + grub_src = os.path.join(deploy_dir, grub_image) + if not os.path.exists(grub_src): + raise WicError("Grub loader %s is not found in %s. " + "Please build grub-efi first" % (grub_image, deploy_dir)) + shutil.copy(grub_src, grub_target) + + if not os.path.isfile(os.path.join(target_dir, "boot.cfg")): + cls.do_configure_grubefi(part, creator, target_dir) else: raise WicError("unrecognized bootimg-efi loader: %s" %
@@ -372,15 +352,6 @@ class IsoImagePlugin(SourcePlugin): except KeyError: raise WicError("bootimg-efi requires a loader, none specified") - if os.path.exists("%s/EFI/BOOT" % isodir): - shutil.rmtree("%s/EFI/BOOT" % isodir) - - shutil.copytree(bootimg_dir+"/EFI/BOOT", isodir+"/EFI/BOOT") - - # If exists, remove cr_workdir/bootimg temporary folder - if os.path.exists("%s/bootimg" % cr_workdir): - shutil.rmtree("%s/bootimg" % cr_workdir) - # Create efi.img that contains bootloader files for EFI booting # if ISODIR didn't exist or didn't contains it if os.path.isfile("%s/efi.img" % img_iso_dir):
Signature · 1 line
2.1.4
PATCH v2 8/8 selftest: fixed 5 wic test cases
In reply to Ed BartoshAdded core-image-minimal -> syslinux dependency to ensure syslinux artifacts are available from core-image-minimal build. This should fix test_iso_image, test_bootloader_config and test_default_output_dir test cases.
Used --native-sysroot in test_image_vars_dir_long and test_image_vars_dir_short test cases to point out to wic-tools native sysrtoot.
--- meta/lib/oeqa/selftest/cases/wic.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-)
meta/lib/oeqa/selftest/cases/wic.py +19 -4
@@ -188,7 +188,8 @@ class Wic(OESelftestTestCase): def test_iso_image(self): """Test creation of hybrid iso image with legacy and EFI boot""" config = 'INITRAMFS_IMAGE = "core-image-minimal-initramfs"\n'\ - 'MACHINE_FEATURES_append = " efi"\n' + 'MACHINE_FEATURES_append = " efi"\n'\ + 'DEPENDS_pn-core-image-minimal += "syslinux"\n' self.append_config(config) bitbake('core-image-minimal') self.remove_config(config)
@@ -217,6 +218,10 @@ class Wic(OESelftestTestCase): @only_for_arch(['i586', 'i686', 'x86_64']) def test_bootloader_config(self): """Test creation of directdisk-bootloader-config image""" + config = 'DEPENDS_pn-core-image-minimal += "syslinux"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) cmd = "wic create directdisk-bootloader-config -e core-image-minimal -o %s" % self.resultdir self.assertEqual(0, runCmd(cmd).status) self.assertEqual(1, len(glob(self.resultdir + "directdisk-bootloader-config-*direct")))
@@ -248,6 +253,10 @@ class Wic(OESelftestTestCase): """Test default output location""" for fname in glob("directdisk-*.direct"): os.remove(fname) + config = 'DEPENDS_pn-core-image-minimal += "syslinux"\n' + self.append_config(config) + bitbake('core-image-minimal') + self.remove_config(config) cmd = "wic create directdisk -e core-image-minimal" self.assertEqual(0, runCmd(cmd).status) self.assertEqual(1, len(glob("directdisk-*.direct")))
@@ -569,10 +578,12 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r """Test image vars directory selection -v option""" image = 'core-image-minimal' imgenvdir = self._get_image_env_path(image) + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "wic-tools") self.assertEqual(0, runCmd("wic create wictestdisk " - "--image-name=%s -v %s -o %s" - % (image, imgenvdir, self.resultdir)).status) + "--image-name=%s -v %s -n %s -o %s" + % (image, imgenvdir, native_sysroot, + self.resultdir)).status) self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) @OETestID(1665)
@@ -580,11 +591,15 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r """Test image vars directory selection --vars option""" image = 'core-image-minimal' imgenvdir = self._get_image_env_path(image) + native_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "wic-tools") + self.assertEqual(0, runCmd("wic create wictestdisk " "--image-name=%s " "--vars %s " + "--native-sysroot %s " "--outdir %s" - % (image, imgenvdir, self.resultdir)).status) + % (image, imgenvdir, native_sysroot, + self.resultdir)).status) self.assertEqual(1, len(glob(self.resultdir + "wictestdisk-*direct"))) @OETestID(1351)
Signature · 1 line
2.1.4