#11283: wic ls
cover #11283: wic ls
Hi,
This is an implementation of new wic subcommand 'wic ls'.
It lists image partitions:
$ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic Num Start End Size Fstype 1 1048576 24438783 23390208 fat16 2 25165824 50315263 25149440 ext4
or directory contents of vfat partitions:
$ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/boot Volume in drive : is boot Volume Serial Number is 66EC-5750 Directory for ::/efi/boot
. <DIR> 2017-05-15 13:00 .. <DIR> 2017-05-15 13:00 grub cfg 639 2017-05-15 13:00 bootx64 efi 571392 2017-05-15 13:00
4 files 572 031 bytes
15 818 752 bytes free
The patchset also fixes and extends sparse_copy API to support copying part of the image (one partition) into another image.
It also contains test case for 'wic ls' functionality.
The following changes since commit ef814cc4db9dedad2ad81d756116af238288c2f9:
create-pull-request: support format-patch options (2017-05-15 14:00:15 +0300)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib ed/wic/wip
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wic/wip
Ed Bartosh (8):
filemap: fix skip logic
filemap: add parameter 'length' to sparse_copy
bootimg-pcbios: make boot image file unique
wic: add wic_init_parser_ls
wic: add help and usage content for 'wic ls'
wic: add 'wic ls' command
engine: implement listing wic images
selftest: add new test case test_wic_ls
meta/lib/oeqa/selftest/wic.py | 18 +++++ scripts/lib/wic/engine.py | 86 +++++++++++++++++++++++- scripts/lib/wic/filemap.py | 26 +++++-- scripts/lib/wic/help.py | 65 ++++++++++++++++++ scripts/lib/wic/plugins/source/bootimg-pcbios.py | 2 +- scripts/wic | 40 +++++++++++ 6 files changed, 228 insertions(+), 9 deletions(-)
--
Regards,
Ed
PATCH 1/8 filemap: fix skip logic
Fixed bug in processing 'skip' parameter:
don't read input file if end of bmap block is less than skip
Simplified logic of positioning to the start of data inside a partially skipped bmap block.
--- scripts/lib/wic/filemap.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
scripts/lib/wic/filemap.py +7 -4
@@ -545,11 +545,14 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): start = first * fmap.block_size end = (last + 1) * fmap.block_size + if skip >= end: + continue + if start < skip < end: - fmap._f_image.seek(skip, os.SEEK_SET) - else: - fmap._f_image.seek(start, os.SEEK_SET) - dst_file.seek(offset + start, os.SEEK_SET) + start = skip + + fmap._f_image.seek(start, os.SEEK_SET) + dst_file.seek(offset + start - skip, os.SEEK_SET) chunk_size = 1024 * 1024 to_read = end - start
Signature · 1 line
2.1.4
PATCH 2/8 filemap: add parameter 'length' to sparse_copy
In reply to Ed BartoshAdded parameter 'length' to specify amount of data to write into destination file. This is useful when only part of source file should be written into destination file.
--- scripts/lib/wic/filemap.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
scripts/lib/wic/filemap.py +12 -3
@@ -530,7 +530,8 @@ def filemap(image, log=None): except ErrorNotSupp: return FilemapSeek(image, log) -def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): +def sparse_copy(src_fname, dst_fname, offset=0, skip=0, + length=0, api=None): """Efficiently copy sparse file to or into another file.""" if not api: api = filemap
@@ -541,6 +542,7 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): dst_file = open(dst_fname, 'wb') dst_file.truncate(os.path.getsize(src_fname)) + written = 0 for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt): start = first * fmap.block_size end = (last + 1) * fmap.block_size
@@ -561,7 +563,14 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): while read < to_read: if read + chunk_size > to_read: chunk_size = to_read - read - chunk = fmap._f_image.read(chunk_size) + size = chunk_size + if length and written + size > length: + size = length - written + chunk = fmap._f_image.read(size) dst_file.write(chunk) - read += chunk_size + read += size + written += size + if written == length: + dst_file.close() + return dst_file.close()
Signature · 1 line
2.1.4
PATCH 3/8 bootimg-pcbios: make boot image file unique
In reply to Ed BartoshPlugin code uses boot.img file name for an image file. If there are two partitions that use bootimg-pcbios wic breaks with an error "file already exists: boot.img"
Made image file name unique by adding wks like number to it to fix the issue.
--- scripts/lib/wic/plugins/source/bootimg-pcbios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/wic/plugins/source/bootimg-pcbios.py +1 -1
@@ -186,7 +186,7 @@ class BootimgPcbiosPlugin(SourcePlugin): extra_blocks, part.mountpoint, blocks) # dosfs image, created by mkdosfs - bootimg = "%s/boot.img" % cr_workdir + bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) dosfs_cmd = "mkdosfs -n boot -S 512 -C %s %d" % (bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot)
Signature · 1 line
2.1.4
PATCH 4/8 wic: add wic_init_parser_ls
In reply to Ed BartoshAdded parser for 'wic ls' command.
--- scripts/wic | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
scripts/wic +25 -0
@@ -35,6 +35,8 @@ import os import sys import argparse import logging + +from collections import namedtuple from distutils import spawn # External modules
@@ -317,6 +319,29 @@ def wic_init_parser_list(subparser): "defined inside the .wks file") return +def imgtype(arg): + """ + Custom type for ArgumentParser + Converts path spec to named tuple: (image, partition, path) + """ + image = arg + part = path = None + if ':' in image: + image, part = image.split(':') + if '/' in part: + part, path = part.split('/', 1) + + if not os.path.isfile(image): + err = "%s is not a regular file or symlink" % image + raise argparse.ArgumentTypeError(err) + + return namedtuple('ImgType', 'image part path')(image, part, path) + +def wic_init_parser_ls(subparser): + subparser.add_argument("path", type=imgtype, + help="image spec: <image>[:<vfat partition>[<path>]]") + subparser.add_argument("-n", "--native-sysroot", + help="path to the native sysroot containing the tools") def wic_init_parser_help(subparser): helpparsers = subparser.add_subparsers(dest='help_topic', help=hlp.wic_usage)
Signature · 1 line
2.1.4
PATCH 5/8 wic: add help and usage content for 'wic ls'
In reply to Ed BartoshAdded wic_ls_help and wic_ls_usage variables to help.py. These variables contain help content that will be used in 'wic ls help' and 'wic ls --help' output.
--- scripts/lib/wic/help.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+)
scripts/lib/wic/help.py +65 -0
@@ -284,6 +284,71 @@ DESCRIPTION details. """ +wic_ls_usage = """ + + List content of a partitioned image + + usage: wic ls <image>[:<vfat partition>[<path>]] [--native-sysroot <path>] + + This command outputs either list of image partitions or directory contents + of vfat partitions. + + See 'wic help ls' for more detailed instructions. + +""" + +wic_ls_help = """ + +NAME + wic ls - List contents of partitioned image or vfat partitions + +SYNOPSIS + wic ls <image> + wic ls <image>:<vfat partition> + wic ls <image>:<vfat partition><path> + wic ls <image>:<vfat partition><path> --native-sysroot <path> + +DESCRIPTION + This command lists either partitions of the image or directory contents + of vfat partitions. + + The first form it lists partitions of the image. + For example: + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic + Num Start End Size Fstype + 1 1048576 24438783 23390208 fat16 + 2 25165824 50315263 25149440 ext4 + + Second and third form list directory content of vfat partition: + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + Volume in drive : is boot + Volume Serial Number is 2DF2-5F02 + Directory for ::/ + + efi <DIR> 2017-05-11 10:54 + startup nsh 26 2017-05-11 10:54 + vmlinuz 6922288 2017-05-11 10:54 + 3 files 6 922 314 bytes + 15 818 752 bytes free + + + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/EFI/boot/ + Volume in drive : is boot + Volume Serial Number is 2DF2-5F02 + Directory for ::/EFI/boot + + . <DIR> 2017-05-11 10:54 + .. <DIR> 2017-05-11 10:54 + grub cfg 679 2017-05-11 10:54 + bootx64 efi 571392 2017-05-11 10:54 + 4 files 572 071 bytes + 15 818 752 bytes free + + The -n option is used to specify the path to the native sysroot + containing the tools(parted and mtools) to use. + +""" + wic_plugins_help = """ NAME
Signature · 1 line
2.1.4
PATCH 6/8 wic: add 'wic ls' command
In reply to Ed BartoshAdded empty 'wic ls' command that does nothing. The functionality will be added by the next commits.
--- scripts/lib/wic/engine.py | 4 ++++ scripts/wic | 15 +++++++++++++++ 2 files changed, 19 insertions(+)
scripts/lib/wic/engine.py +4 -0
@@ -225,6 +225,10 @@ def wic_list(args, scripts_path): return False +def wic_ls(args, native_sysroot): + """List contents of partitioned image or vfat partition.""" + pass + def find_canned(scripts_path, file_name): """ Find a file either by its path or by name in the canned files dir.
scripts/wic +15 -0
@@ -234,6 +234,14 @@ def wic_list_subcommand(args, usage_str): raise WicError("Bad list arguments, exiting") +def wic_ls_subcommand(args, usage_str): + """ + Command-line handling for list content of images. + The real work is done by engine.wic_ls() + """ + engine.wic_ls(args, args.native_sysroot) + + def wic_help_subcommand(args, usage_str): """ Command-line handling for help subcommand to keep the current
@@ -266,6 +274,9 @@ helptopics = { "create": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_create_help], + "ls": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_ls_help], "list": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_list_help]
@@ -359,6 +370,10 @@ subcommands = { hlp.wic_list_usage, hlp.wic_list_help, wic_init_parser_list], + "ls": [wic_ls_subcommand, + hlp.wic_ls_usage, + hlp.wic_ls_help, + wic_init_parser_ls], "help": [wic_help_subcommand, wic_help_topic_usage, hlp.wic_help_help,
Signature · 1 line
2.1.4
PATCH 7/8 engine: implement listing wic images
In reply to Ed BartoshImplemented 'wic ls' functionality:
- list image partitions
- list directory content of vfat partitions
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 84 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-)
scripts/lib/wic/engine.py +82 -2
@@ -30,10 +30,15 @@ import logging import os +import tempfile + +from collections import namedtuple, OrderedDict +from distutils.spawn import find_executable from wic import WicError +from wic.filemap import sparse_copy from wic.pluginbase import PluginMgr -from wic.utils.misc import get_bitbake_var +from wic.utils.misc import get_bitbake_var, exec_cmd logger = logging.getLogger('wic')
@@ -225,9 +230,84 @@ def wic_list(args, scripts_path): return False + +class Disk: + def __init__(self, imagepath, native_sysroot): + self.imagepath = imagepath + self.native_sysroot = native_sysroot + self._partitions = None + self._mdir = None + self._partimages = {} + + # find parted + self.paths = "/bin:/usr/bin:/usr/sbin:/sbin/" + if native_sysroot: + for path in self.paths.split(':'): + self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) + + self.parted = find_executable("parted", self.paths) + if not self.parted: + raise WicError("Can't find executable parted") + + def __del__(self): + for path in self._partimages.values(): + os.unlink(path) + + @property + def partitions(self): + if self._partitions is None: + self._partitions = OrderedDict() + out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) + parttype = namedtuple("Part", "pnum start end size fstype") + for line in out.splitlines()[2:]: + pnum, start, end, size, fstype = line.split(':')[:5] + partition = parttype(pnum, int(start[:-1]), int(end[:-1]), + int(size[:-1]), fstype) + self._partitions[pnum] = partition + + return self._partitions + + @property + def mdir(self): + if self._mdir is None: + self._mdir = find_executable("mdir", self.paths) + if not self._mdir: + raise WicError("Can't find executable mdir") + return self._mdir + + def _get_part_image(self, pnum): + if pnum not in self.partitions: + raise WicError("Partition %s is not in the image") + part = self.partitions[pnum] + if not part.fstype.startswith("fat"): + raise WicError("Not supported fstype: {}".format(part.fstype)) + if pnum not in self._partimages: + tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") + dst_fname = tmpf.name + tmpf.close() + sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) + self._partimages[pnum] = dst_fname + + return self._partimages[pnum] + + def dir(self, pnum, path): + return exec_cmd("{} -i {} ::{}".format(self.mdir, + self._get_part_image(pnum), + path)) + def wic_ls(args, native_sysroot): """List contents of partitioned image or vfat partition.""" - pass + disk = Disk(args.path.image, native_sysroot) + if not args.path.part: + if disk.partitions: + print('Num Start End Size Fstype') + for part in disk.partitions.values(): + print("{:2s} {:12d} {:12d} {:12d} {}".format(\ + part.pnum, part.start, part.end, + part.size, part.fstype)) + else: + path = args.path.path or '/' + print(disk.dir(args.path.part, path)) def find_canned(scripts_path, file_name): """
Signature · 1 line
2.1.4
PATCH 8/8 selftest: add new test case test_wic_ls
In reply to Ed BartoshTested 'wic ls' functionality:
- list of image partitions
- list of directory content of vfat partition
--- meta/lib/oeqa/selftest/wic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
meta/lib/oeqa/selftest/wic.py +18 -0
@@ -825,3 +825,21 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r status, output = qemu.run_serial(cmd) self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output)) self.assertEqual(output, '2') + + def test_wic_ls(self): + """Test listing image content using 'wic ls'""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + images = glob(self.resultdir + "wictestdisk-*.direct") + self.assertEqual(1, len(images)) + + # list partitions + result = runCmd("wic ls %s" % images[0]) + self.assertEqual(0, result.status) + self.assertEqual(3, len(result.output.split('\n'))) + + # list directory content of the first partition + result = runCmd("wic ls %s:1/" % images[0]) + self.assertEqual(0, result.status) + self.assertEqual(6, len(result.output.split('\n')))
Signature · 1 line
2.1.4