#11278 - wic: create an option to use entire disk

index 99912cd..ccd3382 100644
@@ -468,6 +468,46 @@ DESCRIPTION
     containing the tools(parted and mtools) to use.
 """
 
+wic_write_usage = """
+
+ Write image to a device
+
+ usage: wic write <image> <target device> [--expand [rules]] [--native-sysroot <path>]
+
+ This command writes wic image to a target device (USB stick, SD card etc).
+
+ See 'wic help write' for more detailed instructions.
+
+"""
+
+wic_write_help = """
+
+NAME
+    wic write - write wic image to a device
+
+SYNOPSIS
+    wic write <image> <target>
+    wic write <image> <target> --expand auto
+    wic write <image> <target> --expand 1:100M-2:300M
+    wic write <image> <target> --native-sysroot <path>
+
+DESCRIPTION
+    This command writes wic image to a target device (USB stick, SD card etc)
+
+        $ wic write ./tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic /dev/sdb
+
+    The --expand option is used to resize image partitions.
+    --expand auto expands partitions to occupy all free space available on the target device.
+    It's also possible to specify expansion rules in a format
+    <partition>:<size>[-<partition>:<size>...] for one or more partitions.
+    Specifying size 0 will keep partition unmodified.
+    Note: Resizing boot partition can result in non-bootable image for non-EFI images. It is
+    recommended to use size 0 for boot partition to keep image bootable.
+
+    The --native-sysroot option is used to specify the path to the native sysroot
+    containing the tools(parted, resize2fs) to use.
+"""
+
 wic_plugins_help = """
 
 NAME

index 2dc2fd5..d33c9b3 100644
@@ -375,6 +375,12 @@ def wic_rm(args, native_sysroot):
     disk = Disk(args.path.image, native_sysroot)
     disk.remove(args.path.part, args.path.path)
 
+def wic_write(args, native_sysroot):
+    """
+    Write image to a target device.
+    """
+    pass
+
 def find_canned(scripts_path, file_name):
     """
     Find a file either by its path or by name in the canned files dir.

79 lines not on this page · raw message

index d33c9b3..a030ca2 100644
@@ -236,10 +236,6 @@ class Disk:
         self.imagepath = imagepath
         self.native_sysroot = native_sysroot
         self._partitions = None
-        self._mdir = None
-        self._mcopy = None
-        self._mdel = None
-        self._mdeltree = None
         self._partimages = {}
 
         # find parted

@@ -270,30 +266,16 @@ class Disk:
 
         return self._partitions
 
-    def _prop(self, name):
+    def __getattr__(self, name):
         """Get path to the executable in a lazy way."""
-        aname = "_%s" % name
-        if getattr(self, aname) is None:
-            setattr(self, aname, find_executable(name, self.paths))
-            if not getattr(self, aname):
-                raise WicError("Can't find executable {}".format(name))
-        return getattr(self, aname)
-
-    @property
-    def mdir(self):
-        return self._prop('mdir')
-
-    @property
-    def mcopy(self):
-        return self._prop("mcopy")
-
-    @property
-    def mdel(self):
-        return self._prop("mdel")
-
-    @property
-    def mdeltree(self):
-        return self._prop("mdeltree")
+        if name in ("mdir", "mcopy", "mdel", "mdeltree"):
+            aname = "_%s" % name
+            if aname not in self.__dict__:
+                setattr(self, aname, find_executable(name, self.paths))
+                if aname not in self.__dict__:
+                    raise WicError("Can't find executable {}".format(name))
+            return self.__dict__[aname]
+        return self.__dict__[name]
 
     def _get_part_image(self, pnum):
         if pnum not in self.partitions:

index a030ca2..b8fd3ae 100644
@@ -237,6 +237,9 @@ class Disk:
         self.native_sysroot = native_sysroot
         self._partitions = None
         self._partimages = {}
+        self._lsector_size = None
+        self._psector_size = None
+        self._ptable_format = None
 
         # find parted
         self.paths = "/bin:/usr/bin:/usr/sbin:/sbin/"

@@ -258,7 +261,11 @@ class Disk:
             self._partitions = OrderedDict()
             out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
             parttype = namedtuple("Part", "pnum start end size fstype")
-            for line in out.splitlines()[2:]:
+            splitted = out.splitlines()
+            lsector_size, psector_size, self._ptable_format = splitted[1].split(":")[3:6]
+            self._lsector_size = int(lsector_size)
+            self._psector_size = int(psector_size)
+            for line in splitted[2:]:
                 pnum, start, end, size, fstype = line.split(':')[:5]
                 partition = parttype(pnum, int(start[:-1]), int(end[:-1]),
                                      int(size[:-1]), fstype)

index b8fd3ae..83e633d 100644
@@ -267,7 +267,7 @@ class Disk:
             self._psector_size = int(psector_size)
             for line in splitted[2:]:
                 pnum, start, end, size, fstype = line.split(':')[:5]
-                partition = parttype(pnum, int(start[:-1]), int(end[:-1]),
+                partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]),
                                      int(size[:-1]), fstype)
                 self._partitions[pnum] = partition
 

@@ -341,7 +341,7 @@ def wic_ls(args, native_sysroot):
         if disk.partitions:
             print('Num     Start        End          Size      Fstype')
             for part in disk.partitions.values():
-                print("{:2s}  {:12d} {:12d} {:12d}  {}".format(\
+                print("{:2d}  {:12d} {:12d} {:12d}  {}".format(\
                           part.pnum, part.start, part.end,
                           part.size, part.fstype))
     else:

index 83e633d..a6ea362 100644
@@ -232,9 +232,10 @@ def wic_list(args, scripts_path):
 
 
 class Disk:
-    def __init__(self, imagepath, native_sysroot):
+    def __init__(self, imagepath, native_sysroot, fstypes=('fat',)):
         self.imagepath = imagepath
         self.native_sysroot = native_sysroot
+        self.fstypes = fstypes
         self._partitions = None
         self._partimages = {}
         self._lsector_size = None

@@ -288,7 +289,11 @@ class Disk:
         if pnum not in self.partitions:
             raise WicError("Partition %s is not in the image")
         part = self.partitions[pnum]
-        if not part.fstype.startswith("fat"):
+        # check if fstype is supported
+        for fstype in self.fstypes:
+            if part.fstype.startswith(fstype):
+                break
+        else:
             raise WicError("Not supported fstype: {}".format(part.fstype))
         if pnum not in self._partimages:
             tmpf = tempfile.NamedTemporaryFile(prefix="wic-part")

index 428befe..57dd37a 100644
@@ -5,7 +5,8 @@ LICENSE = "MIT"
 DEPENDS = "\
            parted-native syslinux-native gptfdisk-native dosfstools-native \
            mtools-native bmap-tools-native grub-efi-native cdrtools-native \
-           btrfs-tools-native squashfs-tools-native pseudo-native e2fsprogs-native \
+           btrfs-tools-native squashfs-tools-native pseudo-native \
+           e2fsprogs-native util-linux-native \
            "
 DEPENDS_append_x86 = " syslinux grub-efi systemd-boot"
 DEPENDS_append_x86-64 = " syslinux grub-efi systemd-boot"

index a6ea362..3c1f0d3 100644
@@ -276,7 +276,8 @@ class Disk:
 
     def __getattr__(self, name):
         """Get path to the executable in a lazy way."""
-        if name in ("mdir", "mcopy", "mdel", "mdeltree"):
+        if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
+                    "resize2fs", "mkswap", "mkdosfs"):
             aname = "_%s" % name
             if aname not in self.__dict__:
                 setattr(self, aname, find_executable(name, self.paths))

index 3c1f0d3..303f323 100644
@@ -31,6 +31,8 @@
 import logging
 import os
 import tempfile
+import json
+import subprocess
 
 from collections import namedtuple, OrderedDict
 from distutils.spawn import find_executable

@@ -340,6 +342,143 @@ class Disk:
                 raise err
         self._put_part_image(pnum)
 
+    def write(self, target, expand):
+        """Write disk image to the media or file."""
+        def write_sfdisk_script(outf, parts):
+            for key, val in parts['partitiontable'].items():
+                if key in ("partitions", "device", "firstlba", "lastlba"):
+                    continue
+                if key == "id":
+                    key = "label-id"
+                outf.write("{}: {}\n".format(key, val))
+            outf.write("\n")
+            for part in parts['partitiontable']['partitions']:
+                line = ''
+                for name in ('attrs', 'name', 'size', 'type', 'uuid'):
+                    if name == 'size' and part['type'] == 'f':
+                        # don't write size for extended partition
+                        continue
+                    val = part.get(name)
+                    if val:
+                        line += '{}={}, '.format(name, val)
+                if line:
+                    line = line[:-2] # strip ', '
+                if part.get('bootable'):
+                    line += ' ,bootable'
+                outf.write("{}\n".format(line))
+            outf.flush()
+
+        def read_ptable(path):
+            out = exec_cmd("{} -dJ {}".format(self.sfdisk, path))
+            return json.loads(out)
+
+        def write_ptable(parts, target):
+            with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf:
+                write_sfdisk_script(outf, parts)
+                cmd = "{} --no-reread {} < {} 2>/dev/null".format(self.sfdisk, target, outf.name)
+                try:
+                    subprocess.check_output(cmd, shell=True)
+                except subprocess.CalledProcessError as err:
+                    raise WicError("Can't run '{}' command: {}".format(cmd, err))
+
+        if expand is None:
+            sparse_copy(self.imagepath, target)
+        else:
+            # copy first sectors that may contain bootloader
+            sparse_copy(self.imagepath, target, length=2048 * self._lsector_size)
+
+            # copy source partition table to the target
+            parts = read_ptable(self.imagepath)
+            write_ptable(parts, target)
+
+            # get size of unpartitioned space
+            free = None
+            for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines():
+                if line.startswith("Unpartitioned space ") and line.endswith("sectors"):
+                    free = int(line.split()[-2])
+            if free is None:
+                raise WicError("Can't get size of unpartitioned space")
+
+            # calculate expanded partitions sizes
+            sizes = {}
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                if num in expand:
+                    if expand[num] != 0: # don't resize partition if size is set to 0
+                        sectors = expand[num] // self._lsector_size
+                        free -= sectors - part['size']
+                        part['size'] = sectors
+                        sizes[num] = sectors
+                elif part['type'] != 'f':
+                    sizes[num] = -1
+
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                if sizes.get(num) == -1:
+                    part['size'] += free // len(sizes)
+
+            # write resized partition table to the target
+            write_ptable(parts, target)
+
+            # read resized partition table
+            parts = read_ptable(target)
+
+            # copy partitions content
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                pnum = str(num)
+                fstype = self.partitions[pnum].fstype
+
+                # copy unchanged partition
+                if part['size'] == self.partitions[pnum].size // self._lsector_size:
+                    logger.info("copying unchanged partition {}".format(pnum))
+                    sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size)
+                    continue
+
+                # resize or re-create partitions
+                if fstype.startswith('ext') or fstype.startswith('fat') or \
+                   fstype.startswith('linux-swap'):
+
+                    partfname = None
+                    with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf:
+                        partfname = partf.name
+
+                    if fstype.startswith('ext'):
+                        logger.info("resizing ext partition {}".format(pnum))
+                        partimg = self._get_part_image(pnum)
+                        sparse_copy(partimg, partfname)
+                        exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
+                        exec_cmd("{} {} {}s".format(\
+                                 self.resize2fs, partfname, part['size']))
+                    elif fstype.startswith('fat'):
+                        logger.info("copying content of the fat partition {}".format(pnum))
+                        with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir:
+                            # copy content to the temporary directory
+                            cmd = "{} -snompi {} :: {}".format(self.mcopy,
+                                                               self._get_part_image(pnum),
+                                                               tmpdir)
+                            exec_cmd(cmd)
+                            # create new msdos partition
+                            label = part.get("name")
+                            label_str = "-n {}".format(label) if label else ''
+
+                            cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname,
+                                                          part['size'])
+                            exec_cmd(cmd)
+                            # copy content from the temporary directory to the new partition
+                            cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir)
+                            exec_cmd(cmd, as_shell=True)
+                    elif fstype.startswith('linux-swap'):
+                        logger.info("creating swap partition {}".format(pnum))
+                        label = part.get("name")
+                        label_str = "-L {}".format(label) if label else ''
+                        uuid = part.get("uuid")
+                        uuid_str = "-U {}".format(uuid) if uuid else ''
+                        with open(partfname, 'w') as sparse:
+                            os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
+                        exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname))
+                    sparse_copy(partfname, target, seek=part['start'] * self._lsector_size)
+                    os.unlink(partfname)
+                elif part['type'] != 'f':
+                    logger.warn("skipping partition {}: unsupported fstype {}".format(pnum, fstype))
+
 def wic_ls(args, native_sysroot):
     """List contents of partitioned image or vfat partition."""
     disk = Disk(args.path.image, native_sysroot)

@@ -374,7 +513,8 @@ def wic_write(args, native_sysroot):
     """
     Write image to a target device.
     """
-    pass
+    disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'swap'))
+    disk.write(args.target, args.expand)
 
 def find_canned(scripts_path, file_name):
     """

index 303f323..eafc6c7 100644
@@ -254,12 +254,13 @@ class Disk:
         if not self.parted:
             raise WicError("Can't find executable parted")
 
+        self.partitions = self.get_partitions()
+
     def __del__(self):
         for path in self._partimages.values():
             os.unlink(path)
 
-    @property
-    def partitions(self):
+    def get_partitions(self):
         if self._partitions is None:
             self._partitions = OrderedDict()
             out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))

index aaefd4f..aa73ba4 100644
@@ -938,3 +938,56 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
             wksname = os.path.splitext(os.path.basename(wks.name))[0]
             out = glob(self.resultdir + "%s-*direct" % wksname)
             self.assertEqual(1, len(out))
+
+    def test_expand_mbr_image(self):
+        """Test wic write --expand command for mbr image"""
+        # build an image
+        config = 'IMAGE_FSTYPES = "wic"\nWKS_FILE = "directdisk.wks"\n'
+        self.append_config(config)
+        self.assertEqual(0, bitbake('core-image-minimal').status)
+
+        # get path to the image
+        bb_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'MACHINE'])
+        deploy_dir = bb_vars['DEPLOY_DIR_IMAGE']
+        machine = bb_vars['MACHINE']
+        image_path = os.path.join(deploy_dir, 'core-image-minimal-%s.wic' % machine)
+
+        self.remove_config(config)
+
+        try:
+            # expand image to 1G
+            new_image_path = None
+            with NamedTemporaryFile(mode='wb', suffix='.wic.exp',
+                                    dir=deploy_dir, delete=False) as sparse:
+                sparse.truncate(1024 ** 3)
+                new_image_path = sparse.name
+
+            sysroot = get_bb_var('RECIPE_SYSROOT_NATIVE', 'wic-tools')
+            cmd = "wic write -n %s --expand 1:0 %s %s" % (sysroot, image_path, new_image_path)
+            self.assertEqual(0, runCmd(cmd).status)
+
+            # check if partitions are expanded
+            orig = runCmd("wic ls %s -n %s" % (image_path, sysroot))
+            exp = runCmd("wic ls %s -n %s" % (new_image_path, sysroot))
+            orig_sizes = [int(line.split()[3]) for line in orig.output.split('\n')[1:]]
+            exp_sizes = [int(line.split()[3]) for line in exp.output.split('\n')[1:]]
+            self.assertEqual(orig_sizes[0], exp_sizes[0]) # first partition is not resized
+            self.assertTrue(orig_sizes[1] < exp_sizes[1])
+
+            # Check if all free space is partitioned
+            result = runCmd("%s/usr/sbin/sfdisk -F %s" % (sysroot, new_image_path))
+            self.assertTrue("0 B, 0 bytes, 0 sectors" in result.output)
+
+            os.rename(image_path, image_path + '.bak')
+            os.rename(new_image_path, image_path)
+
+            # Check if it boots in qemu
+            with runqemu('core-image-minimal', ssh=False) as qemu:
+                cmd = "ls /etc/"
+                status, output = qemu.run_serial('true')
+                self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output))
+        finally:
+            if os.path.exists(new_image_path):
+                os.unlink(new_image_path)
+            if os.path.exists(image_path + '.bak'):
+                os.rename(image_path + '.bak', image_path)