#10619: refactor wic codebase (continuation)
cover #10619: refactor wic codebase (continuation)
Hi,
This patchset simplifies interfaces of DirectPlugin and underlying set of APIs located in partitionedfs module. It also removes one of two different structures for partitioning info.
The following changes since commit a624cf7f95c8cf4ff764cc997fd1db4601b97dcc:
oeqa/selftest/pkgdata: use m4 instead of bash (2017-02-07 14:50:10 +0000)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib ed/wic/wip
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=ed/wic/wip
Ed Bartosh (9):
wic: partitionedfs: merged __format_disks and create
wic: partitionedfs: get rid of __add_partition
wic: partitionedfs: rename __create_partition and __add_disk
wic: direct: get rid of _get_parts getter
wic: use the same partition object in direct and partitionedfs
wic: make sure layout_partitions is called once
wic: direct: remove unused plugin attributes
wic: direct: remove unused import
wic: remove unused argument scripts_path
scripts/lib/wic/engine.py | 7 +- scripts/lib/wic/partition.py | 2 + scripts/lib/wic/plugins/imager/direct.py | 54 ++------- scripts/lib/wic/utils/partitionedfs.py | 197 ++++++++++++------------------- scripts/wic | 2 +- 5 files changed, 95 insertions(+), 167 deletions(-)
--
Regards,
Ed
PATCH 1/9 wic: partitionedfs: merged __format_disks and create
Private method __format_disks is called only from create method making the code less readable. Merged the code into one method.
--- scripts/lib/wic/utils/partitionedfs.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-)
scripts/lib/wic/utils/partitionedfs.py +5 -10
@@ -238,7 +238,11 @@ class Image(): return exec_native_cmd(cmd, self.native_sysroot) - def __format_disks(self): + def create(self): + for dev in self.disks: + disk = self.disks[dev] + disk['disk'].create() + self.layout_partitions() for dev in self.disks:
@@ -375,12 +379,3 @@ class Image(): partimage = image_file + '.p%d' % part['num'] os.rename(source, partimage) self.partimages.append(partimage) - - def create(self): - for dev in self.disks: - disk = self.disks[dev] - disk['disk'].create() - - self.__format_disks() - - return
Signature · 1 line
2.1.4
PATCH 2/9 wic: partitionedfs: get rid of __add_partition
In reply to Ed Bartosh3 lines long private method __add_partition is called only from add_partition method. Merged them together to increase readability.
--- scripts/lib/wic/utils/partitionedfs.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-)
scripts/lib/wic/utils/partitionedfs.py +4 -10
@@ -80,15 +80,6 @@ class Image(): self.disks[disk_name]['disk'] = disk_obj self.disks[disk_name]['identifier'] = identifier - def __add_partition(self, part): - """ This is a helper function for 'add_partition()' which adds a - partition to the internal list of partitions. """ - - assert not self._partitions_layed_out - - self.partitions.append(part) - self.__add_disk(part['disk_name']) - def add_partition(self, size, disk_name, mountpoint, source_file=None, fstype=None, label=None, fsopts=None, boot=False, align=None, no_table=False, part_type=None, uuid=None, system_id=None):
@@ -117,7 +108,10 @@ class Image(): 'uuid': uuid, # Partition UUID 'system_id': system_id} # Partition system id - self.__add_partition(part) + assert not self._partitions_layed_out + + self.partitions.append(part) + self._add_disk(part['disk_name']) def layout_partitions(self, ptable_format="msdos"): """ Layout the partitions, meaning calculate the position of every
Signature · 1 line
2.1.4
PATCH 3/9 wic: partitionedfs: rename __create_partition and __add_disk
In reply to Ed BartoshRenamed private methods with leading double underscores:__create_partition -> _create_partition __add_disk -> _add_disk
There is no point to have those names mangled, one underscore is enough.
--- scripts/lib/wic/utils/partitionedfs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
scripts/lib/wic/utils/partitionedfs.py +8 -8
@@ -49,7 +49,7 @@ class Image(): self._partitions_layed_out = False self.native_sysroot = native_sysroot - def __add_disk(self, disk_name): + def _add_disk(self, disk_name): """ Add a disk 'disk_name' to the internal list of disks. Note, 'disk_name' is the name of the disk in the target system (e.g., sdb). """
@@ -76,7 +76,7 @@ class Image(): can be added. In case of multiple disks, disk partitions have to be added for each disk separately with 'add_partition()". """ - self.__add_disk(disk_name) + self._add_disk(disk_name) self.disks[disk_name]['disk'] = disk_obj self.disks[disk_name]['identifier'] = identifier
@@ -217,7 +217,7 @@ class Image(): disk['min_size'] *= self.sector_size - def __create_partition(self, device, parttype, fstype, start, size): + def _create_partition(self, device, parttype, fstype, start, size): """ Create a partition on an image described by the 'device' object. """ # Start is included to the size so we need to substract one from the end.
@@ -272,9 +272,9 @@ class Image(): # starts a sector before the first logical partition, # add a sector at the back, so that there is enough # room for all logical partitions. - self.__create_partition(disk['disk'].device, "extended", - None, part['start'] - 1, - disk['offset'] - part['start'] + 1) + self._create_partition(disk['disk'].device, "extended", + None, part['start'] - 1, + disk['offset'] - part['start'] + 1) if part['fstype'] == "swap": parted_fs_type = "linux-swap"
@@ -297,8 +297,8 @@ class Image(): part['mountpoint']) part['size'] -= 1 - self.__create_partition(disk['disk'].device, part['type'], - parted_fs_type, part['start'], part['size']) + self._create_partition(disk['disk'].device, part['type'], + parted_fs_type, part['start'], part['size']) if part['part_type']: msger.debug("partition %d: set type UID to %s" % \
Signature · 1 line
2.1.4
PATCH 4/9 wic: direct: get rid of _get_parts getter
In reply to Ed BartoshReplaced _get_parts getter with direct attribute
access to self.parts
Removed code that implicitly created partition
if there are no partitions mentioned in .wks file
--- scripts/lib/wic/plugins/imager/direct.py | 36 +++++++------------------------- 1 file changed, 8 insertions(+), 28 deletions(-)
scripts/lib/wic/plugins/imager/direct.py +8 -28
@@ -94,6 +94,7 @@ class DirectPlugin(ImagerPlugin): self._disk_format = "direct" self._disk_names = [] self.ptable_format = self.ks.bootloader.ptable + self.parts = self.ks.partitions def do_create(self): """
@@ -140,7 +141,7 @@ class DirectPlugin(ImagerPlugin): with open(fstab_path) as fstab: fstab_lines = fstab.readlines() - if self._update_fstab(fstab_lines, self._get_parts()): + if self._update_fstab(fstab_lines, self.parts): shutil.copyfile(fstab_path, fstab_path + ".orig") with open(fstab_path, "w") as fstab:
@@ -179,22 +180,6 @@ class DirectPlugin(ImagerPlugin): """ self.bootimg_dir = bootimg_dir - def _get_parts(self): - if not self.ks: - raise CreatorError("Failed to get partition info, " - "please check your kickstart setting.") - - # Set a default partition if no partition is given out - if not self.ks.partitions: - partstr = "part / --size 1900 --ondisk sda --fstype=ext3" - args = partstr.split() - part = self.ks.parse(args[1:]) - if part not in self.ks.partitions: - self.ks.partitions.append(part) - - # partitions list from kickstart file - return self.ks.partitions - def _full_path(self, path, name, extention): """ Construct full file path to a file we generate. """ return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
@@ -208,12 +193,10 @@ class DirectPlugin(ImagerPlugin): filesystems from the artifacts directly and combine them into a partitioned image. """ - parts = self._get_parts() - self._image = Image(self.native_sysroot) disk_ids = {} - for num, part in enumerate(parts, 1): + for num, part in enumerate(self.parts, 1): # as a convenience, set source to the boot partition source # instead of forcing it to be set via bootloader --source if not self.ks.bootloader.source and part.mountpoint == "/boot":
@@ -227,11 +210,11 @@ class DirectPlugin(ImagerPlugin): if part.disk not in disk_ids: disk_ids[part.disk] = int.from_bytes(os.urandom(4), 'little') disk_id = disk_ids[part.disk] - part.uuid = '%0x-%02d' % (disk_id, self._get_part_num(num, parts)) + part.uuid = '%0x-%02d' % (disk_id, self._get_part_num(num, self.parts)) fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR")) - for part in parts: + for part in self.parts: # get rootfs size from bitbake variable if it's not set in .ks file if not part.size: # and if rootfs name is specified for the partition
@@ -323,8 +306,6 @@ class DirectPlugin(ImagerPlugin): """ msg = "The new image(s) can be found here:\n" - parts = self._get_parts() - for disk_name in self._image.disks: extension = "direct" + {"gzip": ".gz", "bzip2": ".bz2",
@@ -334,7 +315,7 @@ class DirectPlugin(ImagerPlugin): msg += ' %s\n\n' % full_path msg += 'The following build artifacts were used to create the image(s):\n' - for part in parts: + for part in self.parts: if part.rootfs_dir is None: continue if part.mountpoint == '/':
@@ -357,14 +338,13 @@ class DirectPlugin(ImagerPlugin): Assume partition order same as in wks """ - parts = self._get_parts() - for num, part in enumerate(parts, 1): + for num, part in enumerate(self.parts, 1): if part.mountpoint == "/": if part.uuid: return "PARTUUID=%s" % part.uuid else: suffix = 'p' if part.disk.startswith('mmcblk') else '' - pnum = self._get_part_num(num, parts) + pnum = self._get_part_num(num, self.parts) return "/dev/%s%s%-d" % (part.disk, suffix, pnum) def cleanup(self):
Signature · 1 line
2.1.4
PATCH 5/9 wic: use the same partition object in direct and partitionedfs
In reply to Ed BartoshPartition attributes were copied to the dictionary in partitionedfs code, which makes the code hard to follow.
Used partition object passed from direct.py module as is in partitionedfs.
--- scripts/lib/wic/partition.py | 2 + scripts/lib/wic/plugins/imager/direct.py | 9 +- scripts/lib/wic/utils/partitionedfs.py | 158 ++++++++++++++----------------- 3 files changed, 73 insertions(+), 96 deletions(-)
scripts/lib/wic/partition.py +2 -0
@@ -38,12 +38,14 @@ class Partition(): self.active = args.active self.align = args.align self.disk = args.disk + self.device = None self.extra_space = args.extra_space self.fsopts = args.fsopts self.fstype = args.fstype self.label = args.label self.mountpoint = args.mountpoint self.no_table = args.no_table + self.num = None self.overhead_factor = args.overhead_factor self.part_type = args.part_type self.rootfs_dir = args.rootfs_dir
scripts/lib/wic/plugins/imager/direct.py +1 -8
@@ -237,14 +237,7 @@ class DirectPlugin(ImagerPlugin): part.prepare(self, self.workdir, self.oe_builddir, self.rootfs_dir, self.bootimg_dir, self.kernel_dir, self.native_sysroot) - - self._image.add_partition(part.disk_size, part.disk, - part.mountpoint, part.source_file, - part.fstype, part.label, - fsopts=part.fsopts, boot=part.active, - align=part.align, no_table=part.no_table, - part_type=part.part_type, uuid=part.uuid, - system_id=part.system_id) + self._image.add_partition(part) if fstab_path: shutil.move(fstab_path + ".orig", fstab_path)
scripts/lib/wic/utils/partitionedfs.py +70 -88
286 lines not on this page · raw message
Signature · 1 line
2.1.4
PATCH 6/9 wic: make sure layout_partitions is called once
In reply to Ed BartoshRemoved artificial _partitions_layed_out attribute and unneeded call of layout_partitions method.
--- scripts/lib/wic/utils/partitionedfs.py | 12 ------------ 1 file changed, 12 deletions(-)
scripts/lib/wic/utils/partitionedfs.py +0 -12
@@ -46,7 +46,6 @@ class Image(): self.partimages = [] # Size of a sector used in calculations self.sector_size = SECTOR_SIZE - self._partitions_layed_out = False self.native_sysroot = native_sysroot def _add_disk(self, disk_name):
@@ -58,8 +57,6 @@ class Image(): # We already have this disk return - assert not self._partitions_layed_out - self.disks[disk_name] = \ {'disk': None, # Disk object 'numpart': 0, # Number of allocate partitions
@@ -90,8 +87,6 @@ class Image(): # Converting kB to sectors for parted part.size_sec = part.disk_size * 1024 // self.sector_size - assert not self._partitions_layed_out - self.partitions.append(part) self._add_disk(part.disk)
@@ -102,11 +97,6 @@ class Image(): msger.debug("Assigning %s partitions to disks" % ptable_format) - if self._partitions_layed_out: - return - - self._partitions_layed_out = True - # Go through partitions in the order they are added in .ks file for num in range(len(self.partitions)): part = self.partitions[num]
@@ -219,8 +209,6 @@ class Image(): disk = self.disks[dev] disk['disk'].create() - self.layout_partitions() - for dev in self.disks: disk = self.disks[dev] msger.debug("Initializing partition table for %s" % \
Signature · 1 line
2.1.4
PATCH 7/9 wic: direct: remove unused plugin attributes
In reply to Ed BartoshRemoved unused _disks, _disk_format and _disk_names attributes from DirectPlugin class.
--- scripts/lib/wic/plugins/imager/direct.py | 4 ---- 1 file changed, 4 deletions(-)
scripts/lib/wic/plugins/imager/direct.py +0 -4
@@ -90,9 +90,6 @@ class DirectPlugin(ImagerPlugin): strftime("%Y%m%d%H%M")) self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.') self._image = None - self._disks = {} - self._disk_format = "direct" - self._disk_names = [] self.ptable_format = self.ks.bootloader.ptable self.parts = self.ks.partitions
@@ -249,7 +246,6 @@ class DirectPlugin(ImagerPlugin): msger.debug("Adding disk %s as %s with size %s bytes" \ % (disk_name, full_path, disk['min_size'])) disk_obj = DiskImage(full_path, disk['min_size']) - #self._disks[disk_name] = disk_obj self._image.add_disk(disk_name, disk_obj, disk_ids.get(disk_name)) self._image.create()
Signature · 1 line
2.1.4
PATCH 8/9 wic: direct: remove unused import
In reply to Ed BartoshRemoved unused import of wic.errors module.
--- scripts/lib/wic/plugins/imager/direct.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
scripts/lib/wic/plugins/imager/direct.py +1 -2
@@ -34,7 +34,6 @@ from wic import msger from wic.ksparser import KickStart, KickStartError from wic.plugin import pluginmgr from wic.pluginbase import ImagerPlugin -from wic.utils import errors from wic.utils.errors import CreatorError, ImageError from wic.utils.misc import get_bitbake_var, exec_cmd, exec_native_cmd from wic.utils.partitionedfs import Image
@@ -102,7 +101,7 @@ class DirectPlugin(ImagerPlugin): self.assemble() self.finalize() self.print_info() - except errors.CreatorError: + except CreatorError: raise finally: self.cleanup()
Signature · 1 line
2.1.4
PATCH 9/9 wic: remove unused argument scripts_path
In reply to Ed BartoshThere is no need to pass scripts_path from main wic module down the stack as it's not used there.
Removed scripts_path argument from DirectPlugin class and wic_create function.
--- scripts/lib/wic/engine.py | 7 +++---- scripts/lib/wic/plugins/imager/direct.py | 2 +- scripts/wic | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-)
scripts/lib/wic/engine.py +3 -4
@@ -145,8 +145,8 @@ def list_source_plugins(): print(" %s" % plugin) -def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, native_sysroot, - scripts_path, options): +def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, + native_sysroot, options): """ Create image
@@ -155,7 +155,6 @@ def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, native_sysroot, bootimg_dir - absolute path to the build's boot artifacts directory kernel_dir - absolute path to the build's kernel directory native_sysroot - absolute path to the build's native sysroots dir - scripts_path - absolute path to /scripts dir image_output_dir - dirname to create for image options - wic command line options (debug, bmap, etc)
@@ -195,7 +194,7 @@ def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, native_sysroot, msger.error('Unknown plugin: %s' % pname) plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir, - native_sysroot, scripts_path, oe_builddir, options) + native_sysroot, oe_builddir, options) plugin.do_create()
scripts/lib/wic/plugins/imager/direct.py +1 -1
@@ -68,7 +68,7 @@ class DirectPlugin(ImagerPlugin): name = 'direct' def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir, - native_sysroot, scripts_path, oe_builddir, options): + native_sysroot, oe_builddir, options): try: self.ks = KickStart(wks_file) except KickStartError as err:
scripts/wic +1 -1
@@ -250,7 +250,7 @@ def wic_create_subcommand(args, usage_str): print("Creating image(s)...\n") engine.wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir, - native_sysroot, scripts_path, options) + native_sysroot, options) def wic_list_subcommand(args, usage_str):
Signature · 1 line
2.1.4