#11283 wic ls & cp & rm
cover #11283 wic ls & cp & rm
Hi,
This is a consolidated patchet made of previously sent 'wic ls' 'wic cp' and 'wic rm' patchset. It doesn't depend on generic efi implementation, which made these 3 patchsets pending.
Please, review and merge.
Thanks
The following changes since commit 4a7612c7a12b9a381fb8343ba9586272b889fc15:
buildhistory: skip tests if GitPython module is missing (2017-06-07 16:00:49 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib ed/wip
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wip
Ed Bartosh (25):
filemap: fix skip logic
filemap: add parameter 'length' to sparse_copy
bootimg-pcbios: make boot image file unique
wic: add wic_init_parser_ls
wic: add help and usage content for 'wic ls'
wic: add 'wic ls' command
engine: implement listing wic images
selftest: add new test case test_wic_ls
wic: add wic_init_parser_cp
wic: add help and usage content for 'wic cp'
wic: add 'wic cp' command
wic: add Disk._prop helper
wic: add mcopy property
filemap: change signature of sparse_copy function
filemap: check if dest is written for every block
filemap: calculate dst size correctly
wic: add Disk._put_part_image method
wic: fully implement 'wic cp'
selftest: add test_wic_cp test case
wic: add wic_init_parser_rm
wic: add help and usage content for 'wic rm'
wic: add 'wic rm' command
wic: implement removing files
wic: implement removing directories
selftest: add test_wic_rm test case
meta/lib/oeqa/selftest/cases/wic.py | 96 +++++++++++- scripts/lib/wic/engine.py | 152 ++++++++++++++++++- scripts/lib/wic/filemap.py | 50 ++++-- scripts/lib/wic/help.py | 184 +++++++++++++++++++++++ scripts/lib/wic/plugins/imager/direct.py | 2 +- scripts/lib/wic/plugins/source/bootimg-pcbios.py | 2 +- scripts/wic | 89 +++++++++++ 7 files changed, 562 insertions(+), 13 deletions(-)
Signature · 1 line
2.1.4
PATCH 1/25 filemap: fix skip logic
Fixed bug in processing 'skip' parameter:
don't read input file if end of bmap block is less than skip
Simplified logic of positioning to the start of data inside a partially skipped bmap block.
--- scripts/lib/wic/filemap.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
scripts/lib/wic/filemap.py +7 -4
@@ -545,11 +545,14 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): start = first * fmap.block_size end = (last + 1) * fmap.block_size + if skip >= end: + continue + if start < skip < end: - fmap._f_image.seek(skip, os.SEEK_SET) - else: - fmap._f_image.seek(start, os.SEEK_SET) - dst_file.seek(offset + start, os.SEEK_SET) + start = skip + + fmap._f_image.seek(start, os.SEEK_SET) + dst_file.seek(offset + start - skip, os.SEEK_SET) chunk_size = 1024 * 1024 to_read = end - start
Signature · 1 line
2.1.4
PATCH 2/25 filemap: add parameter 'length' to sparse_copy
In reply to Ed BartoshAdded parameter 'length' to specify amount of data to write into destination file. This is useful when only part of source file should be written into destination file.
--- scripts/lib/wic/filemap.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
scripts/lib/wic/filemap.py +12 -3
@@ -530,7 +530,8 @@ def filemap(image, log=None): except ErrorNotSupp: return FilemapSeek(image, log) -def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): +def sparse_copy(src_fname, dst_fname, offset=0, skip=0, + length=0, api=None): """Efficiently copy sparse file to or into another file.""" if not api: api = filemap
@@ -541,6 +542,7 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): dst_file = open(dst_fname, 'wb') dst_file.truncate(os.path.getsize(src_fname)) + written = 0 for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt): start = first * fmap.block_size end = (last + 1) * fmap.block_size
@@ -561,7 +563,14 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None): while read < to_read: if read + chunk_size > to_read: chunk_size = to_read - read - chunk = fmap._f_image.read(chunk_size) + size = chunk_size + if length and written + size > length: + size = length - written + chunk = fmap._f_image.read(size) dst_file.write(chunk) - read += chunk_size + read += size + written += size + if written == length: + dst_file.close() + return dst_file.close()
Signature · 1 line
2.1.4
PATCH 3/25 bootimg-pcbios: make boot image file unique
In reply to Ed BartoshPlugin code uses boot.img file name for an image file. If there are two partitions that use bootimg-pcbios wic breaks with an error "file already exists: boot.img"
Made image file name unique by adding wks like number to it to fix the issue.
--- scripts/lib/wic/plugins/source/bootimg-pcbios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/wic/plugins/source/bootimg-pcbios.py +1 -1
@@ -186,7 +186,7 @@ class BootimgPcbiosPlugin(SourcePlugin): extra_blocks, part.mountpoint, blocks) # dosfs image, created by mkdosfs - bootimg = "%s/boot.img" % cr_workdir + bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) dosfs_cmd = "mkdosfs -n boot -S 512 -C %s %d" % (bootimg, blocks) exec_native_cmd(dosfs_cmd, native_sysroot)
Signature · 1 line
2.1.4
PATCH 4/25 wic: add wic_init_parser_ls
In reply to Ed BartoshAdded parser for 'wic ls' command.
--- scripts/wic | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+)
scripts/wic +25 -0
@@ -35,6 +35,8 @@ import os import sys import argparse import logging + +from collections import namedtuple from distutils import spawn # External modules
@@ -317,6 +319,29 @@ def wic_init_parser_list(subparser): "defined inside the .wks file") return +def imgtype(arg): + """ + Custom type for ArgumentParser + Converts path spec to named tuple: (image, partition, path) + """ + image = arg + part = path = None + if ':' in image: + image, part = image.split(':') + if '/' in part: + part, path = part.split('/', 1) + + if not os.path.isfile(image): + err = "%s is not a regular file or symlink" % image + raise argparse.ArgumentTypeError(err) + + return namedtuple('ImgType', 'image part path')(image, part, path) + +def wic_init_parser_ls(subparser): + subparser.add_argument("path", type=imgtype, + help="image spec: <image>[:<vfat partition>[<path>]]") + subparser.add_argument("-n", "--native-sysroot", + help="path to the native sysroot containing the tools") def wic_init_parser_help(subparser): helpparsers = subparser.add_subparsers(dest='help_topic', help=hlp.wic_usage)
Signature · 1 line
2.1.4
PATCH 5/25 wic: add help and usage content for 'wic ls'
In reply to Ed BartoshAdded wic_ls_help and wic_ls_usage variables to help.py. These variables contain help content that will be used in 'wic ls help' and 'wic ls --help' output.
--- scripts/lib/wic/help.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+)
scripts/lib/wic/help.py +65 -0
@@ -284,6 +284,71 @@ DESCRIPTION details. """ +wic_ls_usage = """ + + List content of a partitioned image + + usage: wic ls <image>[:<vfat partition>[<path>]] [--native-sysroot <path>] + + This command outputs either list of image partitions or directory contents + of vfat partitions. + + See 'wic help ls' for more detailed instructions. + +""" + +wic_ls_help = """ + +NAME + wic ls - List contents of partitioned image or vfat partitions + +SYNOPSIS + wic ls <image> + wic ls <image>:<vfat partition> + wic ls <image>:<vfat partition><path> + wic ls <image>:<vfat partition><path> --native-sysroot <path> + +DESCRIPTION + This command lists either partitions of the image or directory contents + of vfat partitions. + + The first form it lists partitions of the image. + For example: + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic + Num Start End Size Fstype + 1 1048576 24438783 23390208 fat16 + 2 25165824 50315263 25149440 ext4 + + Second and third form list directory content of vfat partition: + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + Volume in drive : is boot + Volume Serial Number is 2DF2-5F02 + Directory for ::/ + + efi <DIR> 2017-05-11 10:54 + startup nsh 26 2017-05-11 10:54 + vmlinuz 6922288 2017-05-11 10:54 + 3 files 6 922 314 bytes + 15 818 752 bytes free + + + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/EFI/boot/ + Volume in drive : is boot + Volume Serial Number is 2DF2-5F02 + Directory for ::/EFI/boot + + . <DIR> 2017-05-11 10:54 + .. <DIR> 2017-05-11 10:54 + grub cfg 679 2017-05-11 10:54 + bootx64 efi 571392 2017-05-11 10:54 + 4 files 572 071 bytes + 15 818 752 bytes free + + The -n option is used to specify the path to the native sysroot + containing the tools(parted and mtools) to use. + +""" + wic_plugins_help = """ NAME
Signature · 1 line
2.1.4
PATCH 6/25 wic: add 'wic ls' command
In reply to Ed BartoshAdded empty 'wic ls' command that does nothing. The functionality will be added by the next commits.
--- scripts/lib/wic/engine.py | 4 ++++ scripts/wic | 15 +++++++++++++++ 2 files changed, 19 insertions(+)
scripts/lib/wic/engine.py +4 -0
@@ -225,6 +225,10 @@ def wic_list(args, scripts_path): return False +def wic_ls(args, native_sysroot): + """List contents of partitioned image or vfat partition.""" + pass + def find_canned(scripts_path, file_name): """ Find a file either by its path or by name in the canned files dir.
scripts/wic +15 -0
@@ -234,6 +234,14 @@ def wic_list_subcommand(args, usage_str): raise WicError("Bad list arguments, exiting") +def wic_ls_subcommand(args, usage_str): + """ + Command-line handling for list content of images. + The real work is done by engine.wic_ls() + """ + engine.wic_ls(args, args.native_sysroot) + + def wic_help_subcommand(args, usage_str): """ Command-line handling for help subcommand to keep the current
@@ -266,6 +274,9 @@ helptopics = { "create": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_create_help], + "ls": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_ls_help], "list": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_list_help]
@@ -359,6 +370,10 @@ subcommands = { hlp.wic_list_usage, hlp.wic_list_help, wic_init_parser_list], + "ls": [wic_ls_subcommand, + hlp.wic_ls_usage, + hlp.wic_ls_help, + wic_init_parser_ls], "help": [wic_help_subcommand, wic_help_topic_usage, hlp.wic_help_help,
Signature · 1 line
2.1.4
PATCH 7/25 engine: implement listing wic images
In reply to Ed BartoshImplemented 'wic ls' functionality:
- list image partitions
- list directory content of vfat partitions
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 84 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-)
scripts/lib/wic/engine.py +82 -2
@@ -30,10 +30,15 @@ import logging import os +import tempfile + +from collections import namedtuple, OrderedDict +from distutils.spawn import find_executable from wic import WicError +from wic.filemap import sparse_copy from wic.pluginbase import PluginMgr -from wic.utils.misc import get_bitbake_var +from wic.utils.misc import get_bitbake_var, exec_cmd logger = logging.getLogger('wic')
@@ -225,9 +230,84 @@ def wic_list(args, scripts_path): return False + +class Disk: + def __init__(self, imagepath, native_sysroot): + self.imagepath = imagepath + self.native_sysroot = native_sysroot + self._partitions = None + self._mdir = None + self._partimages = {} + + # find parted + self.paths = "/bin:/usr/bin:/usr/sbin:/sbin/" + if native_sysroot: + for path in self.paths.split(':'): + self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) + + self.parted = find_executable("parted", self.paths) + if not self.parted: + raise WicError("Can't find executable parted") + + def __del__(self): + for path in self._partimages.values(): + os.unlink(path) + + @property + def partitions(self): + if self._partitions is None: + self._partitions = OrderedDict() + out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath)) + parttype = namedtuple("Part", "pnum start end size fstype") + for line in out.splitlines()[2:]: + pnum, start, end, size, fstype = line.split(':')[:5] + partition = parttype(pnum, int(start[:-1]), int(end[:-1]), + int(size[:-1]), fstype) + self._partitions[pnum] = partition + + return self._partitions + + @property + def mdir(self): + if self._mdir is None: + self._mdir = find_executable("mdir", self.paths) + if not self._mdir: + raise WicError("Can't find executable mdir") + return self._mdir + + def _get_part_image(self, pnum): + if pnum not in self.partitions: + raise WicError("Partition %s is not in the image") + part = self.partitions[pnum] + if not part.fstype.startswith("fat"): + raise WicError("Not supported fstype: {}".format(part.fstype)) + if pnum not in self._partimages: + tmpf = tempfile.NamedTemporaryFile(prefix="wic-part") + dst_fname = tmpf.name + tmpf.close() + sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size) + self._partimages[pnum] = dst_fname + + return self._partimages[pnum] + + def dir(self, pnum, path): + return exec_cmd("{} -i {} ::{}".format(self.mdir, + self._get_part_image(pnum), + path)) + def wic_ls(args, native_sysroot): """List contents of partitioned image or vfat partition.""" - pass + disk = Disk(args.path.image, native_sysroot) + if not args.path.part: + if disk.partitions: + print('Num Start End Size Fstype') + for part in disk.partitions.values(): + print("{:2s} {:12d} {:12d} {:12d} {}".format(\ + part.pnum, part.start, part.end, + part.size, part.fstype)) + else: + path = args.path.path or '/' + print(disk.dir(args.path.part, path)) def find_canned(scripts_path, file_name): """
Signature · 1 line
2.1.4
PATCH 8/25 selftest: add new test case test_wic_ls
In reply to Ed BartoshTested 'wic ls' functionality:
- list of image partitions
- list of directory content of vfat partition
--- meta/lib/oeqa/selftest/cases/wic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
meta/lib/oeqa/selftest/cases/wic.py +20 -0
@@ -791,3 +791,23 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r # 8 blocks is 4K (physical sector size) self.assertEqual(dest_stat.st_blocks, 8) os.unlink(dest) + + def test_wic_ls(self): + """Test listing image content using 'wic ls'""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + images = glob(self.resultdir + "wictestdisk-*.direct") + self.assertEqual(1, len(images)) + + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + + # list partitions + result = runCmd("wic ls %s -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertEqual(3, len(result.output.split('\n'))) + + # list directory content of the first partition + result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertEqual(6, len(result.output.split('\n')))
Signature · 1 line
2.1.4
PATCH 9/25 wic: add wic_init_parser_cp
In reply to Ed BartoshAdd parser for 'wic cp' subcommand and a custom argument type.
--- scripts/wic | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
scripts/wic +16 -0
@@ -341,6 +341,8 @@ def imgtype(arg): image, part = image.split(':') if '/' in part: part, path = part.split('/', 1) + if not path: + path = '/' if not os.path.isfile(image): err = "%s is not a regular file or symlink" % image
@@ -354,6 +356,20 @@ def wic_init_parser_ls(subparser): subparser.add_argument("-n", "--native-sysroot", help="path to the native sysroot containing the tools") +def imgpathtype(arg): + img = imgtype(arg) + if img.part is None: + raise argparse.ArgumentTypeError("partition number is not specified") + return img + +def wic_init_parser_cp(subparser): + subparser.add_argument("src", + help="source spec") + subparser.add_argument("dest", type=imgpathtype, + help="image spec: <image>:<vfat partition>[<path>]") + subparser.add_argument("-n", "--native-sysroot", + help="path to the native sysroot containing the tools") + def wic_init_parser_help(subparser): helpparsers = subparser.add_subparsers(dest='help_topic', help=hlp.wic_usage) for helptopic in helptopics:
Signature · 1 line
2.1.4
PATCH 10/25 wic: add help and usage content for 'wic cp'
In reply to Ed BartoshAdded wic_cp_help and wic_cp_usage variables to help.py. These variables contain help content that will be used in 'wic cp help' and 'wic cp --help' output.
[YOCTO #11283]
--- scripts/lib/wic/help.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+)
scripts/lib/wic/help.py +62 -0
@@ -349,6 +349,68 @@ DESCRIPTION """ +wic_cp_usage = """ + + Copy files and directories to the vfat partitions + + usage: wic cp <src> <image>:<vfat partition>[<path>] [--native-sysroot <path>] + + This command copies local files or directories to the vfat partitions of partitioned + image. + + See 'wic help cp' for more detailed instructions. + +""" + +wic_cp_help = """ + +NAME + wic cp - copy files and directories to the vfat partitions + +SYNOPSIS + wic cp <src> <image>:<vfat partition> + wic cp <src> <image>:<vfat partition><path> + wic cp <src> <image>:<vfat partition><path> --native-sysroot <path> + +DESCRIPTION + This command copies files and directories to the vfat partition of the + wic image. + + The first form of it copies file or directory to the root directory of + the vfat partition: + $ wic cp test.wks tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + Volume in drive : is boot + Volume Serial Number is DB4C-FD4C + Directory for ::/ + + efi <DIR> 2017-05-24 18:15 + loader <DIR> 2017-05-24 18:15 + startup nsh 26 2017-05-24 18:15 + vmlinuz 6926384 2017-05-24 18:15 + test wks 628 2017-05-24 21:22 + 5 files 6 927 038 bytes + 15 677 440 bytes free + + The second form of the command copies file or directory to the specified directory + on the vfat partition: + $ wic cp test tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/ + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/eti/ + Volume in drive : is boot + Volume Serial Number is DB4C-FD4C + Directory for ::/efi + + . <DIR> 2017-05-24 18:15 + .. <DIR> 2017-05-24 18:15 + boot <DIR> 2017-05-24 18:15 + test <DIR> 2017-05-24 21:27 + 4 files 0 bytes + 15 675 392 bytes free + + The -n option is used to specify the path to the native sysroot + containing the tools(parted and mtools) to use. +""" + wic_plugins_help = """ NAME
Signature · 1 line
2.1.4
PATCH 11/25 wic: add 'wic cp' command
In reply to Ed BartoshAdded empty 'wic cp' command that does nothing. The functionality will be added by the next commits.
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 7 +++++++ scripts/wic | 13 +++++++++++++ 2 files changed, 20 insertions(+)
scripts/lib/wic/engine.py +7 -0
@@ -309,6 +309,13 @@ def wic_ls(args, native_sysroot): path = args.path.path or '/' print(disk.dir(args.path.part, path)) +def wic_cp(args, native_sysroot): + """ + Copy local file or directory to the vfat partition of + partitioned image. + """ + pass + def find_canned(scripts_path, file_name): """ Find a file either by its path or by name in the canned files dir.
scripts/wic +13 -0
@@ -241,6 +241,12 @@ def wic_ls_subcommand(args, usage_str): """ engine.wic_ls(args, args.native_sysroot) +def wic_cp_subcommand(args, usage_str): + """ + Command-line handling for copying files/dirs to images. + The real work is done by engine.wic_cp() + """ + engine.wic_cp(args, args.native_sysroot) def wic_help_subcommand(args, usage_str): """
@@ -277,6 +283,9 @@ helptopics = { "ls": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_ls_help], + "cp": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_cp_help], "list": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_list_help]
@@ -390,6 +399,10 @@ subcommands = { hlp.wic_ls_usage, hlp.wic_ls_help, wic_init_parser_ls], + "cp": [wic_cp_subcommand, + hlp.wic_cp_usage, + hlp.wic_cp_help, + wic_init_parser_cp], "help": [wic_help_subcommand, wic_help_topic_usage, hlp.wic_help_help,
Signature · 1 line
2.1.4
PATCH 12/25 wic: add Disk._prop helper
In reply to Ed BartoshAdded generic helper to use in property methods to access commands in a lazy manner.
--- scripts/lib/wic/engine.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
scripts/lib/wic/engine.py +10 -5
@@ -267,13 +267,18 @@ class Disk: return self._partitions + def _prop(self, name): + """Get path to the executable in a lazy way.""" + aname = "_%s" % name + if getattr(self, aname) is None: + setattr(self, aname, find_executable(name, self.paths)) + if not getattr(self, aname): + raise WicError("Can't find executable {}".format(name)) + return getattr(self, aname) + @property def mdir(self): - if self._mdir is None: - self._mdir = find_executable("mdir", self.paths) - if not self._mdir: - raise WicError("Can't find executable mdir") - return self._mdir + return self._prop('mdir') def _get_part_image(self, pnum): if pnum not in self.partitions:
Signature · 1 line
2.1.4
PATCH 13/25 wic: add mcopy property
In reply to Ed BartoshAdded property that points to the mcopy executable.
--- scripts/lib/wic/engine.py | 5 +++++ 1 file changed, 5 insertions(+)
scripts/lib/wic/engine.py +5 -0
@@ -237,6 +237,7 @@ class Disk: self.native_sysroot = native_sysroot self._partitions = None self._mdir = None + self._mcopy = None self._partimages = {} # find parted
@@ -280,6 +281,10 @@ class Disk: def mdir(self): return self._prop('mdir') + @property + def mcopy(self): + return self._prop("mcopy") + def _get_part_image(self, pnum): if pnum not in self.partitions: raise WicError("Partition %s is not in the image")
Signature · 1 line
2.1.4
PATCH 14/25 filemap: change signature of sparse_copy function
In reply to Ed BartoshRenamed parameter offset->skip to match names of dd parameters.
Changed affected sparse_copy calls.
Added explanation of the parameters to docstring.
--- scripts/lib/wic/filemap.py | 15 ++++++++++++--- scripts/lib/wic/plugins/imager/direct.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-)
scripts/lib/wic/filemap.py +12 -3
@@ -530,9 +530,18 @@ def filemap(image, log=None): except ErrorNotSupp: return FilemapSeek(image, log) -def sparse_copy(src_fname, dst_fname, offset=0, skip=0, +def sparse_copy(src_fname, dst_fname, skip=0, seek=0, length=0, api=None): - """Efficiently copy sparse file to or into another file.""" + """ + Efficiently copy sparse file to or into another file. + + src_fname: path to source file + dst_fname: path to destination file + skip: skip N bytes at thestart of src + seek: seek N bytes from the start of dst + length: read N bytes from src and write them to dst + api: FilemapFiemap or FilemapSeek object + """ if not api: api = filemap fmap = api(src_fname)
@@ -554,7 +563,7 @@ def sparse_copy(src_fname, dst_fname, offset=0, skip=0, start = skip fmap._f_image.seek(start, os.SEEK_SET) - dst_file.seek(offset + start - skip, os.SEEK_SET) + dst_file.seek(seek + start - skip, os.SEEK_SET) chunk_size = 1024 * 1024 to_read = end - start
scripts/lib/wic/plugins/imager/direct.py +1 -1
@@ -550,7 +550,7 @@ class PartitionedImage(): source = part.source_file if source: # install source_file contents into a partition - sparse_copy(source, self.path, part.start * self.sector_size) + sparse_copy(source, self.path, seek=part.start * self.sector_size) logger.debug("Installed %s in partition %d, sectors %d-%d, " "size %d sectors", source, part.num, part.start,
Signature · 1 line
2.1.4
PATCH 15/25 filemap: check if dest is written for every block
In reply to Ed BartoshIf lenght parameter is provided to sparse_copy call it's mandatory to check if the output file is fully written after reading unmapped block from input file.
If it's not done then sparse_copy can write more data than specified length.
--- scripts/lib/wic/filemap.py | 7 +++++++ 1 file changed, 7 insertions(+)
scripts/lib/wic/filemap.py +7 -0
@@ -563,6 +563,13 @@ def sparse_copy(src_fname, dst_fname, skip=0, seek=0, start = skip fmap._f_image.seek(start, os.SEEK_SET) + + written += start - skip - written + if length and written >= length: + dst_file.seek(seek + length, os.SEEK_SET) + dst_file.close() + return + dst_file.seek(seek + start - skip, os.SEEK_SET) chunk_size = 1024 * 1024
Signature · 1 line
2.1.4
PATCH 16/25 filemap: calculate dst size correctly
In reply to Ed BartoshFixed calculation of the dst file size using skip, seek and length parameters. Current code does it incorrectly which causes sparse_copy API to create unnecessary big output files.
--- scripts/lib/wic/filemap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)
scripts/lib/wic/filemap.py +5 -1
@@ -549,7 +549,11 @@ def sparse_copy(src_fname, dst_fname, skip=0, seek=0, dst_file = open(dst_fname, 'r+b') except IOError: dst_file = open(dst_fname, 'wb') - dst_file.truncate(os.path.getsize(src_fname)) + if length: + dst_size = length + seek + else: + dst_size = os.path.getsize(src_fname) + seek - skip + dst_file.truncate(dst_size) written = 0 for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt):
Signature · 1 line
2.1.4
PATCH 17/25 wic: add Disk._put_part_image method
In reply to Ed BartoshThis method copies partition image into the wic image. It will be used in 'wic cp' and 'wic rm' subcommands to copy changed partition back into wic image.
--- scripts/lib/wic/engine.py | 5 +++++ 1 file changed, 5 insertions(+)
scripts/lib/wic/engine.py +5 -0
@@ -300,6 +300,11 @@ class Disk: return self._partimages[pnum] + def _put_part_image(self, pnum): + """Put partition image into partitioned image.""" + sparse_copy(self._partimages[pnum], self.imagepath, + seek=self.partitions[pnum].start) + def dir(self, pnum, path): return exec_cmd("{} -i {} ::{}".format(self.mdir, self._get_part_image(pnum),
Signature · 1 line
2.1.4
PATCH 18/25 wic: fully implement 'wic cp'
In reply to Ed BartoshAdded implementation of Disk.copy method and wic_cp function that copies files/directories to the vfat partition of the partitioned image.
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
scripts/lib/wic/engine.py +10 -1
@@ -310,6 +310,14 @@ class Disk: self._get_part_image(pnum), path)) + def copy(self, src, pnum, path): + """Copy partition image into wic image.""" + cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, + self._get_part_image(pnum), + src, path) + exec_cmd(cmd) + self._put_part_image(pnum) + def wic_ls(args, native_sysroot): """List contents of partitioned image or vfat partition.""" disk = Disk(args.path.image, native_sysroot)
@@ -329,7 +337,8 @@ def wic_cp(args, native_sysroot): Copy local file or directory to the vfat partition of partitioned image. """ - pass + disk = Disk(args.dest.image, native_sysroot) + disk.copy(args.src, args.dest.part, args.dest.path) def find_canned(scripts_path, file_name): """
Signature · 1 line
2.1.4
PATCH 19/25 selftest: add test_wic_cp test case
In reply to Ed BartoshAdded test case for "wic cp" functionality.
- copy file to vfat partition
- copy directory to vfat partition
--- meta/lib/oeqa/selftest/cases/wic.py | 46 ++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-)
meta/lib/oeqa/selftest/cases/wic.py +45 -1
@@ -28,7 +28,7 @@ import sys import unittest from glob import glob -from shutil import rmtree +from shutil import rmtree, copy from functools import wraps, lru_cache from tempfile import NamedTemporaryFile
@@ -811,3 +811,47 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) self.assertEqual(0, result.status) self.assertEqual(6, len(result.output.split('\n'))) + + def test_wic_cp(self): + """Test copy files and directories to the the wic image.""" + self.assertEqual(0, runCmd("wic create wictestdisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + images = glob(self.resultdir + "wictestdisk-*.direct") + self.assertEqual(1, len(images)) + + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + + # list directory content of the first partition + result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertEqual(6, len(result.output.split('\n'))) + + with NamedTemporaryFile("w", suffix=".wic-cp") as testfile: + testfile.write("test") + + # copy file to the partition + result = runCmd("wic cp %s %s:1/ -n %s" % (testfile.name, images[0], sysroot)) + self.assertEqual(0, result.status) + + # check if file is there + result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertEqual(7, len(result.output.split('\n'))) + self.assertTrue(os.path.basename(testfile.name) in result.output) + + # prepare directory + testdir = os.path.join(self.resultdir, 'wic-test-cp-dir') + testsubdir = os.path.join(testdir, 'subdir') + os.makedirs(os.path.join(testsubdir)) + copy(testfile.name, testdir) + + # copy directory to the partition + result = runCmd("wic cp %s %s:1/ -n %s" % (testdir, images[0], sysroot)) + self.assertEqual(0, result.status) + + # check if directory is there + result = runCmd("wic ls %s:1/ -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertEqual(8, len(result.output.split('\n'))) + self.assertTrue(os.path.basename(testdir) in result.output)
Signature · 1 line
2.1.4
PATCH 20/25 wic: add wic_init_parser_rm
In reply to Ed BartoshAdd parser for 'wic rm' subcommand.
--- scripts/wic | 6 ++++++ 1 file changed, 6 insertions(+)
scripts/wic +6 -0
@@ -379,6 +379,12 @@ def wic_init_parser_cp(subparser): subparser.add_argument("-n", "--native-sysroot", help="path to the native sysroot containing the tools") +def wic_init_parser_rm(subparser): + subparser.add_argument("path", type=imgpathtype, + help="path: <image>:<vfat partition><path>") + subparser.add_argument("-n", "--native-sysroot", + help="path to the native sysroot containing the tools") + def wic_init_parser_help(subparser): helpparsers = subparser.add_subparsers(dest='help_topic', help=hlp.wic_usage) for helptopic in helptopics:
Signature · 1 line
2.1.4
PATCH 21/25 wic: add help and usage content for 'wic rm'
In reply to Ed BartoshAdded wic_rm_help and wic_rm_usage variables to help.py. These variables contain help content that will be used in 'wic rm help' and 'wic rm --help' output.
--- scripts/lib/wic/help.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-)
scripts/lib/wic/help.py +58 -1
@@ -395,7 +395,7 @@ DESCRIPTION The second form of the command copies file or directory to the specified directory on the vfat partition: $ wic cp test tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/ - $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/eti/ + $ wic ls tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/efi/ Volume in drive : is boot Volume Serial Number is DB4C-FD4C Directory for ::/efi
@@ -411,6 +411,63 @@ DESCRIPTION containing the tools(parted and mtools) to use. """ +wic_rm_usage = """ + + Remove files or directories from the vfat partitions + + usage: wic rm <image>:<vfat partition><path> [--native-sysroot <path>] + + This command removes files or directories from the vfat partitions of partitioned + image. + + See 'wic help rm' for more detailed instructions. + +""" + +wic_rm_help = """ + +NAME + wic rm - remove files or directories from the vfat partitions + +SYNOPSIS + wic rm <src> <image>:<vfat partition><path> + wic rm <src> <image>:<vfat partition><path> --native-sysroot <path> + +DESCRIPTION + This command removes files or directories from the vfat partition of the + wic image: + + $ wic ls ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + Volume in drive : is boot + Volume Serial Number is 11D0-DE21 + Directory for ::/ + + libcom32 c32 186500 2017-06-02 15:15 + libutil c32 24148 2017-06-02 15:15 + syslinux cfg 209 2017-06-02 15:15 + vesamenu c32 27104 2017-06-02 15:15 + vmlinuz 6926384 2017-06-02 15:15 + 5 files 7 164 345 bytes + 16 582 656 bytes free + + $ wic rm ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1/libutil.c32 + + $ wic ls ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic:1 + Volume in drive : is boot + Volume Serial Number is 11D0-DE21 + Directory for ::/ + + libcom32 c32 186500 2017-06-02 15:15 + syslinux cfg 209 2017-06-02 15:15 + vesamenu c32 27104 2017-06-02 15:15 + vmlinuz 6926384 2017-06-02 15:15 + 4 files 7 140 197 bytes + 16 607 232 bytes free + + The -n option is used to specify the path to the native sysroot + containing the tools(parted and mtools) to use. +""" + wic_plugins_help = """ NAME
Signature · 1 line
2.1.4
PATCH 22/25 wic: add 'wic rm' command
In reply to Ed BartoshAdded empty 'wic rm' command that does nothing. The functionality will be added by the next commits.
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 7 +++++++ scripts/wic | 14 ++++++++++++++ 2 files changed, 21 insertions(+)
scripts/lib/wic/engine.py +7 -0
@@ -340,6 +340,13 @@ def wic_cp(args, native_sysroot): disk = Disk(args.dest.image, native_sysroot) disk.copy(args.src, args.dest.part, args.dest.path) +def wic_rm(args, native_sysroot): + """ + Remove files or directories from the vfat partition of + partitioned image. + """ + pass + def find_canned(scripts_path, file_name): """ Find a file either by its path or by name in the canned files dir.
scripts/wic +14 -0
@@ -248,6 +248,13 @@ def wic_cp_subcommand(args, usage_str): """ engine.wic_cp(args, args.native_sysroot) +def wic_rm_subcommand(args, usage_str): + """ + Command-line handling for removing files/dirs from images. + The real work is done by engine.wic_rm() + """ + engine.wic_rm(args, args.native_sysroot) + def wic_help_subcommand(args, usage_str): """ Command-line handling for help subcommand to keep the current
@@ -286,6 +293,9 @@ helptopics = { "cp": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_cp_help], + "rm": [wic_help_topic_subcommand, + wic_help_topic_usage, + hlp.wic_rm_help], "list": [wic_help_topic_subcommand, wic_help_topic_usage, hlp.wic_list_help]
@@ -409,6 +419,10 @@ subcommands = { hlp.wic_cp_usage, hlp.wic_cp_help, wic_init_parser_cp], + "rm": [wic_rm_subcommand, + hlp.wic_rm_usage, + hlp.wic_rm_help, + wic_init_parser_rm], "help": [wic_help_subcommand, wic_help_topic_usage, hlp.wic_help_help,
Signature · 1 line
2.1.4
PATCH 23/25 wic: implement removing files
In reply to Ed BartoshAdded implementation of Disk.del method and wic_r function that removes files from the vfat partition using mdel utility.
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)
scripts/lib/wic/engine.py +15 -1
@@ -238,6 +238,7 @@ class Disk: self._partitions = None self._mdir = None self._mcopy = None + self._mdel = None self._partimages = {} # find parted
@@ -285,6 +286,10 @@ class Disk: def mcopy(self): return self._prop("mcopy") + @property + def mdel(self): + return self._prop("mdel") + def _get_part_image(self, pnum): if pnum not in self.partitions: raise WicError("Partition %s is not in the image")
@@ -318,6 +323,14 @@ class Disk: exec_cmd(cmd) self._put_part_image(pnum) + def remove(self, pnum, path): + """Remove files/dirs from the partition.""" + cmd = "{} -i {} ::{}".format(self.mdel, + self._get_part_image(pnum), + path) + exec_cmd(cmd) + self._put_part_image(pnum) + def wic_ls(args, native_sysroot): """List contents of partitioned image or vfat partition.""" disk = Disk(args.path.image, native_sysroot)
@@ -345,7 +358,8 @@ def wic_rm(args, native_sysroot): Remove files or directories from the vfat partition of partitioned image. """ - pass + disk = Disk(args.path.image, native_sysroot) + disk.remove(args.path.part, args.path.path) def find_canned(scripts_path, file_name): """
Signature · 1 line
2.1.4
PATCH 24/25 wic: implement removing directories
In reply to Ed BartoshAdded support for removing directories using mdeltree utility to Disk.del method
[YOCTO #11283]
--- scripts/lib/wic/engine.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
scripts/lib/wic/engine.py +18 -4
@@ -239,6 +239,7 @@ class Disk: self._mdir = None self._mcopy = None self._mdel = None + self._mdeltree = None self._partimages = {} # find parted
@@ -290,6 +291,10 @@ class Disk: def mdel(self): return self._prop("mdel") + @property + def mdeltree(self): + return self._prop("mdeltree") + def _get_part_image(self, pnum): if pnum not in self.partitions: raise WicError("Partition %s is not in the image")
@@ -325,10 +330,19 @@ class Disk: def remove(self, pnum, path): """Remove files/dirs from the partition.""" - cmd = "{} -i {} ::{}".format(self.mdel, - self._get_part_image(pnum), - path) - exec_cmd(cmd) + partimg = self._get_part_image(pnum) + cmd = "{} -i {} ::{}".format(self.mdel, partimg, path) + try: + exec_cmd(cmd) + except WicError as err: + if "not found" in str(err) or "non empty" in str(err): + # mdel outputs 'File ... not found' or 'directory .. non empty" + # try to use mdeltree as path could be a directory + cmd = "{} -i {} ::{}".format(self.mdeltree, + partimg, path) + exec_cmd(cmd) + else: + raise err self._put_part_image(pnum) def wic_ls(args, native_sysroot):
Signature · 1 line
2.1.4
PATCH 25/25 selftest: add test_wic_rm test case
In reply to Ed BartoshAdded test case for "wic rm" functionality.
- remove file from vfat partition
- remove directory from vfat partition
--- meta/lib/oeqa/selftest/cases/wic.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)
meta/lib/oeqa/selftest/cases/wic.py +30 -0
@@ -855,3 +855,33 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r self.assertEqual(0, result.status) self.assertEqual(8, len(result.output.split('\n'))) self.assertTrue(os.path.basename(testdir) in result.output) + + def test_wic_rm(self): + """Test removing files and directories from the the wic image.""" + self.assertEqual(0, runCmd("wic create mkefidisk " + "--image-name=core-image-minimal " + "-D -o %s" % self.resultdir).status) + images = glob(self.resultdir + "mkefidisk-*.direct") + self.assertEqual(1, len(images)) + + sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools') + + # list directory content of the first partition + result = runCmd("wic ls %s:1 -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertTrue('\nbzimage ' in result.output) + self.assertTrue('\nefi <DIR> ' in result.output) + + # remove file + result = runCmd("wic rm %s:1/bzimage -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + + # remove directory + result = runCmd("wic rm %s:1/efi -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + + # check if they're removed + result = runCmd("wic ls %s:1 -n %s" % (images[0], sysroot)) + self.assertEqual(0, result.status) + self.assertFalse('\nbzimage ' in result.output) + self.assertFalse('\nefi <DIR> ' in result.output)
Signature · 1 line
2.1.4