wic/engine: fix copying directories into wic image with ext* partition
PATCH 1/1 wic/engine: fix copying directories into wic image with ext* partition
From: "Dragomir, Daniel" <[email protected]>
wic uses debugfs to write on ext* partitions, but debugfs can only write to the current working directory and it cannot copy complete directory trees. Running 'wic ls' on a copied directory show this:
-l: Ext2 inode is not a directory
Fix this by creating a command list for debugfs (-f parameter) when recursive parsing the host directory in order to create a similar directory structure (mkdir) and copy files (write) on each level into the destination directory from the wic's ext* partition.
--- scripts/lib/wic/engine.py | 63 ++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 14 deletions(-)
scripts/lib/wic/engine.py +49 -14
@@ -327,29 +327,64 @@ class Disk: path)) def copy(self, src, dest): - """Copy partition image into wic image.""" - pnum = dest.part if isinstance(src, str) else src.part + """Copy files or directories to/from the vfat or ext* partition.""" + pnum = dest.part if isinstance(src, str) else src.part + partimg = self._get_part_image(pnum) if self.partitions[pnum].fstype.startswith('ext'): - if isinstance(src, str): - cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ - format(os.path.dirname(dest.path), src, os.path.basename(src), - self.debugfs, self._get_part_image(pnum)) - else: # copy from wic - # run both dump and rdump to support both files and directory + if isinstance(src, str): # host to image case + if os.path.isdir(src): + base = os.path.abspath(src) + base_parent = os.path.dirname(base) + cmds = [] + made = set() + + for root, dirs, files in os.walk(base): + for fname in files: + host_file = os.path.join(root, fname) + rel = os.path.relpath(host_file, base_parent) + dest_file = os.path.join(dest.path, rel) + dest_dir = os.path.dirname(dest_file) + + # create dir structure (mkdir -p) + parts = dest_dir.strip('/').split('/') + cur = '' + for p in parts: + cur = cur + '/' + p + if cur not in made: + cmds.append(f'mkdir "{cur}"') + made.add(cur) + + cmds.append(f'write "{host_file}" "{dest_file}"') + + # write script to a temp file + with tempfile.NamedTemporaryFile(mode='w', delete=False, + prefix='wic-debugfs-') as tf: + for line in cmds: + tf.write(line + '\n') + scriptname = tf.name + + cmd = f"{self.debugfs} -w -f {scriptname} {partimg}" + + else: # single file + cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\ + format(os.path.dirname(dest.path), src, + os.path.basename(src), self.debugfs, partimg) + + else: # image to host case cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\ format(os.path.dirname(src.path), src.path, - dest, src.path, dest, self.debugfs, - self._get_part_image(pnum)) + dest, src.path, dest, self.debugfs, partimg) + else: # fat if isinstance(src, str): cmd = "{} -i {} -snop {} ::{}".format(self.mcopy, - self._get_part_image(pnum), - src, dest.path) + partimg, + src, dest.path) else: cmd = "{} -i {} -snop ::{} {}".format(self.mcopy, - self._get_part_image(pnum), - src.path, dest) + partimg, + src.path, dest) exec_cmd(cmd, as_shell=True) self._put_part_image(pnum)
Signature · 1 line
2.25.1
PATCH 1/1 oeqa/selftest/wic: test recursive dir copy on ext partitions
From: "Dragomir, Daniel" <[email protected]>
Extend the wic selftests to cover recursive directory copying into ext partitions.
Previously, copying a directory into an ext partition could appear to succeed, but attempting to access the directory contents would fail with:
-l: Ext2 inode is not a directory
This was fixed in commit 4fc3b42774 ("wic/engine: fix copying directories into wic image with ext* partition").
This test now verifies that directories copied with "wic cp" into an ext4 partition:
- are created with correct inode types
- can be listed recursively with "wic ls"
- preserve files and subdirectories
- can be copied back out of the image without data loss
A simple directory structure is used in this test:
wic-test-cp-ext-dir/
├── topfile.txt
└── subdir/
└── subfile.txt--- meta/lib/oeqa/selftest/cases/wic.py | 65 +++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+)
meta/lib/oeqa/selftest/cases/wic.py +65 -0
@@ -12,6 +12,7 @@ import os import sys import unittest import hashlib +import filecmp from glob import glob from shutil import rmtree, copy
@@ -1662,6 +1663,70 @@ class ModifyTests(WicTestCase): runCmd("wic cp %s:2/etc/fstab %s -n %s" % (images[0], testfile.name, sysroot)) self.assertTrue(os.stat(testfile.name).st_size > 0, msg="Filesize not as expected %s" % os.stat(testfile.name).st_size) + # prepare directory structure + testdir = os.path.join(self.resultdir, "wic-test-cp-ext-dir") + testsubdir = os.path.join(testdir, "subdir") + os.makedirs(testsubdir) + + # add a file in the top-level of the directory + src_file = os.path.join(testdir, "topfile.txt") + with open(src_file, "w") as f: + f.write("top-level\n") + + # add file in the subdir + src_subfile = os.path.join(testsubdir, "subfile.txt") + with open(src_subfile, "w") as f: + f.write("sub-level\n") + + # copy directory to the partition root + runCmd("wic cp %s %s:2/ -n %s" % (testdir, images[0], sysroot)) + basedir = os.path.basename(testdir) + + # check if directory is there at partition root + result = runCmd("wic ls %s:2/ -n %s" % (images[0], sysroot)) + root_entries = set(line.split()[-1] for line in result.output.split('\n') if line) + self.assertIn(basedir, root_entries, msg="Expected directory not present at root: %s" % root_entries) + + # list INSIDE the copied directory + result = runCmd("wic ls %s:2/%s/ -n %s" % (images[0], basedir, sysroot)) + self.assertEqual(0, result.status, + msg="wic ls inside copied directory failed. Output:\n%s" % result.output) + self.assertNotIn("Ext2 inode is not a directory", result.output, + msg="Regression detected (inode not a directory). Output:\n%s" % result.output) + + inside_entries = set(line.split()[-1] for line in result.output.split('\n') if line) + self.assertTrue(set(["subdir", "topfile.txt"]).issubset(inside_entries), + msg="Expected entries missing inside dir: %s" % inside_entries) + + # list inside the subdir + result = runCmd("wic ls %s:2/%s/subdir/ -n %s" % (images[0], basedir, sysroot)) + self.assertEqual(0, result.status, + msg="wic ls inside copied subdir failed. Output:\n%s" % result.output) + self.assertNotIn("Ext2 inode is not a directory", result.output, + msg="Regression detected (inode not a directory). Output:\n%s" % result.output) + + sub_entries = set(line.split()[-1] for line in result.output.split('\n') if line) + self.assertIn("subfile.txt", sub_entries, msg="Expected file missing in subdir: %s" % sub_entries) + + # copy directory from the partition and compare with original + outparent = os.path.join(self.resultdir, "wic-test-cp-ext-out") + os.makedirs(outparent) + runCmd("wic cp %s:2/%s %s -n %s" % (images[0], basedir, outparent, sysroot)) + + copied_dir = os.path.join(outparent, basedir) + self.assertTrue(os.path.isdir(copied_dir), msg="Copied-back directory not created: %s" % copied_dir) + + copied_file = os.path.join(copied_dir, "topfile.txt") + copied_subfile = os.path.join(copied_dir, "subdir", "subfile.txt") + + self.assertTrue(os.path.isfile(copied_file), msg="Missing copied-back file: %s" % copied_file) + self.assertTrue(os.path.isfile(copied_subfile), msg="Missing copied-back subfile: %s" % copied_subfile) + + self.assertTrue(filecmp.cmp(src_file, copied_file, shallow=False), + msg="topfile.txt differs after round-trip copy") + self.assertTrue(filecmp.cmp(src_subfile, copied_subfile, shallow=False), + msg="subfile.txt differs after round-trip copy") + def test_wic_rm_ext(self): """Test removing files from the ext partition."""
Signature · 1 line
2.25.1