Skip to contentSkip to conversation

Add support for erofs filesystems

PATCH Superseded Richard Weinberger 3 participants 8 messages May 25, 2021

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Add support for erofs filesystems

Since Linux 5.4 a new read-only filesystem is available, erofs. Compared to squashfs it offers much better read performance with and without compression enabled.
It suppports two optional compressors, lz4 and lz4hc.

From the mkfs.erofs man page:
EROFS is a new enhanced lightweight linux read-only filesystem with modern designs (eg. no buffer head, reduced metadata, inline xattrs/data, etc.) for scenarios which need high-performance read-only requirements, e.g. Android OS for smartphones and LIVECDs. It also provides fixed-sized output compression support, which improves storage density, keeps relatively higher compression ratios, which is more useful to achieve high performance for embedded devices with limited memory since it has unnoticable memory overhead and page cache thrashing.

This commit adds support for three new filesystem targets:
erofs: erofs without compression
erofs-lz4: erofs with lz4 compresssion enabled
erofs-lz4hc: erofs with lz4hc compression enabled

---
 meta/classes/image_types.bbclass               |  9 +++++++++
 .../erofs-utils/erofs-utils_git.bb             | 18 ++++++++++++++++++
 scripts/lib/wic/help.py                        |  1 +
 scripts/lib/wic/ksparser.py                    |  7 ++++++-
 scripts/lib/wic/partition.py                   | 16 +++++++++++++---
 scripts/lib/wic/plugins/source/rawcopy.py      |  6 +++---
 6 files changed, 50 insertions(+), 7 deletions(-)
 create mode 100644 meta/recipes-devtools/erofs-utils/erofs-utils_git.bb
meta/classes/image_types.bbclass +9 -0
index 8028691405a7..4e95e0e2935e 100644
@@ -108,6 +108,11 @@ IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME
 IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo"
 IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lz4 ${EXTRA_IMAGECMD} -noappend -comp lz4"
 
+IMAGE_CMD_erofs = "mkfs.erofs ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4 = "mkfs.erofs -zlz4 ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4 ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4hc = "mkfs.erofs -zlz4hc ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4hc ${IMAGE_ROOTFS}"
+
+
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
 IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
@@ -243,6 +248,9 @@ do_image_ubi[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_ubifs[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_multiubi[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_f2fs[depends] += "f2fs-tools-native:do_populate_sysroot"
+do_image_erofs[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4hc[depends] += "erofs-utils-native:do_populate_sysroot"
 
 # This variable is available to request which values are suitable for IMAGE_FSTYPES
 IMAGE_TYPES = " \
@@ -261,6 +269,7 @@ IMAGE_TYPES = " \
     wic wic.gz wic.bz2 wic.lzma wic.zst \
     container \
     f2fs \
+    erofs \
 "
 
 # Compression is a special case of conversion. The old variable
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +18 -0
new file mode 100644
index 000000000000..6d35d3f2fa6d
@@ -0,0 +1,18 @@
+SUMMARY = "Tools for erofs filesystems"
+LICENSE = "GPLv2"
+SECTION = "base"
+LIC_FILES_CHKSUM = "file://COPYING;md5=94fa01670a2a8f2d3ab2de15004e0848"
+
+PV = "1.2.1"
+SRCREV = "d1f4953edfcf4f51c71ba91586e21fc6ce9f6db9"
+SRC_URI = "git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git"
+
+S = "${WORKDIR}/git"
+
+DEPENDS = "util-linux-libuuid lz4"
+
+inherit pkgconfig autotools
+
+EXTRA_OECONF = "--enable-lz4 --disable-fuse"
+
+BBCLASSEXTEND = "native nativesdk"
scripts/lib/wic/help.py +1 -0
index bd3a2b97dfaf..991e5094bb84 100644
@@ -930,6 +930,7 @@ DESCRIPTION
              ext4
              btrfs
              squashfs
+             erofs
              swap
 
          --fsoptions: Specifies a free-form string of options to be
scripts/lib/wic/ksparser.py +6 -1
index 3eb669da39ca..7a4cc83af556 100644
@@ -157,7 +157,8 @@ class KickStart():
         part.add_argument('--fsoptions', dest='fsopts')
         part.add_argument('--fstype', default='vfat',
                           choices=('ext2', 'ext3', 'ext4', 'btrfs',
-                                   'squashfs', 'vfat', 'msdos', 'swap'))
+                                   'squashfs', 'vfat', 'msdos', 'erofs',
+                                   'swap'))
         part.add_argument('--mkfs-extraopts', default='')
         part.add_argument('--label')
         part.add_argument('--use-label', action='store_true')
@@ -229,6 +230,10 @@ class KickStart():
                                 err = "%s:%d: SquashFS does not support LABEL" \
                                        % (confpath, lineno)
                                 raise KickStartError(err)
+                        # erofs does not support filesystem labels
+                        if parsed.fstype == 'erofs' and parsed.label:
+                            err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+                            raise KickStartError(err)
                         if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
                             if parsed.fsuuid:
                                 if parsed.fsuuid.upper().startswith('0X'):
scripts/lib/wic/partition.py +13 -3
index 76d144d12d30..e0b2c5bdf29b 100644
@@ -141,9 +141,9 @@ class Partition():
                                             native_sysroot)
                 self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
             else:
-                if self.fstype == 'squashfs':
-                    raise WicError("It's not possible to create empty squashfs "
-                                   "partition '%s'" % (self.mountpoint))
+                if self.fstype in ('squashfs', 'erofs'):
+                    raise WicError("It's not possible to create empty %s "
+                                   "partition '%s'" % (self.fstype, self.mountpoint))
 
                 rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
                                              self.lineno, self.fstype)
@@ -369,6 +369,16 @@ class Partition():
                        (rootfs_dir, rootfs, extraopts)
         exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
 
+    def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a erofs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or ''
+        erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+                       (extraopts, self.fsuuid, rootfs, rootfs_dir)
+        exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+
     def prepare_empty_partition_ext(self, rootfs, oe_builddir,
                                     native_sysroot):
         """
scripts/lib/wic/plugins/source/rawcopy.py +3 -3
index 3c4997d8ba5e..fa7b1eb8ac75 100644
@@ -29,9 +29,9 @@ class RawCopyPlugin(SourcePlugin):
             cmd = 'btrfs filesystem label %s %s' % (dst, label)
         elif fstype == 'swap':
             cmd = 'mkswap -L %s %s' % (label, dst)
-        elif fstype == 'squashfs':
-            raise WicError("It's not possible to update a squashfs "
-                           "filesystem label '%s'" % (label))
+        elif fstype in ('squashfs', 'erofs'):
+            raise WicError("It's not possible to update a %s "
+                           "filesystem label '%s'" % (fstype, label))
         else:
             raise WicError("Cannot update filesystem label: "
                            "Unknown fstype: '%s'" % (fstype))
Signature · 1 line
2.26.2
#1 lore raw

PATCH 1/1 Document erofs filesystem targets

---
 documentation/ref-manual/kickstart.rst | 2 ++
 documentation/ref-manual/variables.rst | 3 +++
 2 files changed, 5 insertions(+)
documentation/ref-manual/kickstart.rst +2 -0
index 8308ffff5b72..2c8b724ff9ac 100644
@@ -116,6 +116,8 @@ the ``part`` and ``partition`` commands:
 
    -  ``squashfs``
 
+   -  ``erofs``
+
    -  ``swap``
 
 -  ``--fsoptions``: Specifies a free-form string of options to be used
documentation/ref-manual/variables.rst +3 -0
index df6413b68ae9..d4ea4f399921 100644
@@ -3293,6 +3293,9 @@ system and gives an overview of their function and contents.
       - jffs2
       - jffs2.sum
       - multiubi
+      - erofs
+      - erofs-lz4
+      - erofs-lz4hc
       - squashfs
       - squashfs-lz4
       - squashfs-lzo
Signature · 1 line
2.26.2
#2 lore raw

Re: [docs] [yocto-docs][PATCH] Document erofs filesystem targets

Hi Richard,

Quoted 11 lines · Richard Weinberger

---
documentation/ref-manual/kickstart.rst | 2 ++
documentation/ref-manual/variables.rst | 3 +++

2 files changed, 5 insertions(+)
documentation/ref-manual/kickstart.rst +0 -0
index 8308ffff5b72..2c8b724ff9ac 100644

documentation/ref-manual/kickstart.rst the ``part`` and ``partition`` commands: · line 122

Quoted 7 lines
documentation/ref-manual/kickstart.rst +2 -0
@@ -116,6 +116,8 @@ the ``part`` and ``partition`` commands:
 
   -  ``squashfs``
 
+   -  ``erofs``
+
   -  ``swap``
 

Please order alphabetically.

documentation/ref-manual/variables.rst system and gives an overview of their function and contents. · line 3298

Quoted 12 lines
documentation/ref-manual/kickstart.rst +0 -1
-  ``--fsoptions``: Specifies a free-form string of options to be used
documentation/ref-manual/variables.rst +3 -0
index df6413b68ae9..d4ea4f399921 100644
@@ -3293,6 +3293,9 @@ system and gives an overview of their function and contents.
      - jffs2
      - jffs2.sum
      - multiubi
+      - erofs
+      - erofs-lz4
+      - erofs-lz4hc

Ditto.

Thanks for the patch,
Cheers,
Quentin

Quoted 6 lines
documentation/ref-manual/variables.rst +0 -0
      - squashfs
      - squashfs-lz4
      - squashfs-lzo
Signature · 1 line
2.26.2
Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.
#3 lore raw

PATCH 1/1 Add support for erofs image types

In reply to Richard Weinberger
---
 lib/toaster/orm/models.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
lib/toaster/orm/models.py +4 -3
index 7f7e922adecc..517774baa0b1 100644
@@ -968,9 +968,10 @@ class Target_Image_File(models.Model):
         'btrfs', 'container', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma',
         'cpio.xz', 'cramfs', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma',
         'ext3', 'ext3.gz', 'ext4', 'ext4.gz', 'f2fs', 'hddimg', 'iso', 'jffs2',
-        'jffs2.sum', 'multiubi', 'squashfs', 'squashfs-lz4', 'squashfs-lzo',
-        'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
-        'ubifs', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
+        'jffs2.sum', 'multiubi', 'erofs', 'erofs-lz4', 'erofs-lz4hc',
+        'squashfs', 'squashfs-lz4', 'squashfs-lzo', 'squashfs-xz', 'tar',
+        'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi', 'ubifs', 'wic',
+        'wic.bz2', 'wic.gz', 'wic.lzma'
     }
 
     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Signature · 1 line
2.26.2
#4 lore raw
In reply to Richard Weinberger
Quoted 78 lines · Richard Weinberger

Since Linux 5.4 a new read-only filesystem is available, erofs. Compared to squashfs it offers much better read performance with and without compression enabled.
It suppports two optional compressors, lz4 and lz4hc.

From the mkfs.erofs man page:
EROFS is a new enhanced lightweight linux read-only filesystem with modern designs (eg. no buffer head, reduced metadata, inline xattrs/data, etc.) for scenarios which need high-performance read-only requirements, e.g. Android OS for smartphones and LIVECDs. It also provides fixed-sized output compression support, which improves storage density, keeps relatively higher compression ratios, which is more useful to achieve high performance for embedded devices with limited memory since it has unnoticable memory overhead and page cache thrashing.

This commit adds support for three new filesystem targets:
erofs: erofs without compression
erofs-lz4: erofs with lz4 compresssion enabled
erofs-lz4hc: erofs with lz4hc compression enabled

---
  meta/classes/image_types.bbclass               |  9 +++++++++
  .../erofs-utils/erofs-utils_git.bb             | 18 ++++++++++++++++++
  scripts/lib/wic/help.py                        |  1 +
  scripts/lib/wic/ksparser.py                    |  7 ++++++-
  scripts/lib/wic/partition.py                   | 16 +++++++++++++---
  scripts/lib/wic/plugins/source/rawcopy.py      |  6 +++---
  6 files changed, 50 insertions(+), 7 deletions(-)

create mode 100644 meta/recipes-devtools/erofs-utils/erofs-utils_git.bb

meta/classes/image_types.bbclass +9 -0
index 8028691405a7..4e95e0e2935e 100644
@@ -108,6 +108,11 @@ IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME
  IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo"
  IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lz4 ${EXTRA_IMAGECMD} -noappend -comp lz4"
 
+IMAGE_CMD_erofs = "mkfs.erofs ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4 = "mkfs.erofs -zlz4 ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4 ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4hc = "mkfs.erofs -zlz4hc ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4hc ${IMAGE_ROOTFS}"
+
+
  IMAGE_CMD_TAR ?= "tar"
  # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
  IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
@@ -243,6 +248,9 @@ do_image_ubi[depends] += "mtd-utils-native:do_populate_sysroot"
  do_image_ubifs[depends] += "mtd-utils-native:do_populate_sysroot"
  do_image_multiubi[depends] += "mtd-utils-native:do_populate_sysroot"
  do_image_f2fs[depends] += "f2fs-tools-native:do_populate_sysroot"
+do_image_erofs[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4hc[depends] += "erofs-utils-native:do_populate_sysroot"
 
  # This variable is available to request which values are suitable for IMAGE_FSTYPES
  IMAGE_TYPES = " \
@@ -261,6 +269,7 @@ IMAGE_TYPES = " \
      wic wic.gz wic.bz2 wic.lzma wic.zst \
      container \
      f2fs \
+    erofs \
  "
 
  # Compression is a special case of conversion. The old variable
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +6 -0
new file mode 100644
index 000000000000..6d35d3f2fa6d
@@ -0,0 +1,18 @@
+SUMMARY = "Tools for erofs filesystems"
+LICENSE = "GPLv2"
+SECTION = "base"
+LIC_FILES_CHKSUM = "file://COPYING;md5=94fa01670a2a8f2d3ab2de15004e0848"
+
+PV = "1.2.1"

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb line 16

Quoted 10 lines
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +10 -0
+SRCREV = "d1f4953edfcf4f51c71ba91586e21fc6ce9f6db9"
+SRC_URI = "git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git"
+
+S = "${WORKDIR}/git"
+
+DEPENDS = "util-linux-libuuid lz4"
+
+inherit pkgconfig autotools
+
+EXTRA_OECONF = "--enable-lz4 --disable-fuse"

In the commit message it's mentioned that compression is optional, still here lz4 is enabled by default - I would rather see these compression switches configurable via PACKAGECONFIG

Quoted 95 lines
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +2 -0
+
+BBCLASSEXTEND = "native nativesdk"
scripts/lib/wic/help.py +1 -0
index bd3a2b97dfaf..991e5094bb84 100644
@@ -930,6 +930,7 @@ DESCRIPTION
               ext4
               btrfs
               squashfs
+             erofs
               swap
 
           --fsoptions: Specifies a free-form string of options to be
scripts/lib/wic/ksparser.py +6 -1
index 3eb669da39ca..7a4cc83af556 100644
@@ -157,7 +157,8 @@ class KickStart():
          part.add_argument('--fsoptions', dest='fsopts')
          part.add_argument('--fstype', default='vfat',
                            choices=('ext2', 'ext3', 'ext4', 'btrfs',
-                                   'squashfs', 'vfat', 'msdos', 'swap'))
+                                   'squashfs', 'vfat', 'msdos', 'erofs',
+                                   'swap'))
          part.add_argument('--mkfs-extraopts', default='')
          part.add_argument('--label')
          part.add_argument('--use-label', action='store_true')
@@ -229,6 +230,10 @@ class KickStart():
                                  err = "%s:%d: SquashFS does not support LABEL" \
                                         % (confpath, lineno)
                                  raise KickStartError(err)
+                        # erofs does not support filesystem labels
+                        if parsed.fstype == 'erofs' and parsed.label:
+                            err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+                            raise KickStartError(err)
                          if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
                              if parsed.fsuuid:
                                  if parsed.fsuuid.upper().startswith('0X'):
scripts/lib/wic/partition.py +13 -3
index 76d144d12d30..e0b2c5bdf29b 100644
@@ -141,9 +141,9 @@ class Partition():
                                              native_sysroot)
                  self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
              else:
-                if self.fstype == 'squashfs':
-                    raise WicError("It's not possible to create empty squashfs "
-                                   "partition '%s'" % (self.mountpoint))
+                if self.fstype in ('squashfs', 'erofs'):
+                    raise WicError("It's not possible to create empty %s "
+                                   "partition '%s'" % (self.fstype, self.mountpoint))
 
                  rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
                                               self.lineno, self.fstype)
@@ -369,6 +369,16 @@ class Partition():
                         (rootfs_dir, rootfs, extraopts)
          exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
 
+    def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a erofs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or ''
+        erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+                       (extraopts, self.fsuuid, rootfs, rootfs_dir)
+        exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+
      def prepare_empty_partition_ext(self, rootfs, oe_builddir,
                                      native_sysroot):
          """
scripts/lib/wic/plugins/source/rawcopy.py +3 -3
index 3c4997d8ba5e..fa7b1eb8ac75 100644
@@ -29,9 +29,9 @@ class RawCopyPlugin(SourcePlugin):
              cmd = 'btrfs filesystem label %s %s' % (dst, label)
          elif fstype == 'swap':
              cmd = 'mkswap -L %s %s' % (label, dst)
-        elif fstype == 'squashfs':
-            raise WicError("It's not possible to update a squashfs "
-                           "filesystem label '%s'" % (label))
+        elif fstype in ('squashfs', 'erofs'):
+            raise WicError("It's not possible to update a %s "
+                           "filesystem label '%s'" % (fstype, label))
          else:
              raise WicError("Cannot update filesystem label: "
                             "Unknown fstype: '%s'" % (fstype))
#5 lore raw

Konrad,

----- Ursprüngliche Mail -----

Quoted 6 lines

Von: "Konrad Weihmann" <[email protected]>

Quoted 1 line

+EXTRA_OECONF = "--enable-lz4 --disable-fuse"

In the commit message it's mentioned that compression is optional, still here lz4 is enabled by default - I would rather see these compression switches configurable via PACKAGECONFIG

Good point! Will send a v2 in a jiffy.

Thanks,
//richard

#6 lore raw
In reply to Richard Weinberger

Hi Richard,

Quoted 59 lines · Richard Weinberger

Since Linux 5.4 a new read-only filesystem is available, erofs. Compared to squashfs it offers much better read performance with and without compression enabled.
It suppports two optional compressors, lz4 and lz4hc.

Quoted 1 line

From the mkfs.erofs man page:

EROFS is a new enhanced lightweight linux read-only filesystem with modern designs (eg. no buffer head, reduced metadata, inline xattrs/data, etc.) for scenarios which need high-performance read-only requirements, e.g. Android OS for smartphones and LIVECDs. It also provides fixed-sized output compression support, which improves storage density, keeps relatively higher compression ratios, which is more useful to achieve high performance for embedded devices with limited memory since it has unnoticable memory overhead and page cache thrashing.

This commit adds support for three new filesystem targets:
erofs: erofs without compression
erofs-lz4: erofs with lz4 compresssion enabled
erofs-lz4hc: erofs with lz4hc compression enabled

---

meta/classes/image_types.bbclass               |  9 +++++++++
.../erofs-utils/erofs-utils_git.bb             | 18 ++++++++++++++++++
scripts/lib/wic/help.py                        |  1 +
scripts/lib/wic/ksparser.py                    |  7 ++++++-
scripts/lib/wic/partition.py                   | 16 +++++++++++++---
scripts/lib/wic/plugins/source/rawcopy.py      |  6 +++---
6 files changed, 50 insertions(+), 7 deletions(-)

create mode 100644 meta/recipes-devtools/erofs-utils/erofs-utils_git.bb

meta/classes/image_types.bbclass +0 -0
index 8028691405a7..4e95e0e2935e 100644
@@ -108,6 +108,11 @@ IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME

IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo" IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lz4 ${EXTRA_IMAGECMD} -noappend -comp lz4"

meta/classes/image_types.bbclass +5 -0
+IMAGE_CMD_erofs = "mkfs.erofs ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4 = "mkfs.erofs -zlz4 ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4 ${IMAGE_ROOTFS}"
+IMAGE_CMD_erofs-lz4hc = "mkfs.erofs -zlz4hc ${EXTRA_IMAGECMD} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.erofs-lz4hc ${IMAGE_ROOTFS}"
+
+

IMAGE_CMD_TAR ?= "tar"

# ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
meta/classes/image_types.bbclass +0 -0
@@ -243,6 +248,9 @@ do_image_ubi[depends] += "mtd-utils-native:do_populate_sysroot"

do_image_ubifs[depends] += "mtd-utils-native:do_populate_sysroot" do_image_multiubi[depends] += "mtd-utils-native:do_populate_sysroot" do_image_f2fs[depends] += "f2fs-tools-native:do_populate_sysroot"

meta/classes/image_types.bbclass +3 -0
+do_image_erofs[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4[depends] += "erofs-utils-native:do_populate_sysroot"
+do_image_erofs_lz4hc[depends] += "erofs-utils-native:do_populate_sysroot"

# This variable is available to request which values are suitable for IMAGE_FSTYPES IMAGE_TYPES = " \

meta/classes/image_types.bbclass IMAGE_TYPES = " \ · line 272

Quoted 4 lines
meta/classes/image_types.bbclass +1 -0
@@ -261,6 +269,7 @@ IMAGE_TYPES = " \
    wic wic.gz wic.bz2 wic.lzma wic.zst \
    container \
    f2fs \
+    erofs \

I think you're supposed to have erofs erofs-lz4 erofs-lz4hc here.

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb

Quoted 8 lines

"

# Compression is a special case of conversion. The old variable

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +0 -0
new file mode 100644
index 000000000000..6d35d3f2fa6d

Any reason for using _git over _1.2.1?

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb line 2

Quoted 3 lines
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +2 -0
@@ -0,0 +1,18 @@
+SUMMARY = "Tools for erofs filesystems"
+LICENSE = "GPLv2"

GPL-2.0-or-later actually.

Thanks for the patch,
Cheers,
Quentin

Quoted 107 lines
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +16 -0
+SECTION = "base"
+LIC_FILES_CHKSUM = "file://COPYING;md5=94fa01670a2a8f2d3ab2de15004e0848"
+
+PV = "1.2.1"
+SRCREV = "d1f4953edfcf4f51c71ba91586e21fc6ce9f6db9"
+SRC_URI = "git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git"
+
+S = "${WORKDIR}/git"
+
+DEPENDS = "util-linux-libuuid lz4"
+
+inherit pkgconfig autotools
+
+EXTRA_OECONF = "--enable-lz4 --disable-fuse"
+
+BBCLASSEXTEND = "native nativesdk"
scripts/lib/wic/help.py +1 -0
index bd3a2b97dfaf..991e5094bb84 100644
@@ -930,6 +930,7 @@ DESCRIPTION
             ext4
             btrfs
             squashfs
+             erofs
             swap
 
         --fsoptions: Specifies a free-form string of options to be
scripts/lib/wic/ksparser.py +6 -1
index 3eb669da39ca..7a4cc83af556 100644
@@ -157,7 +157,8 @@ class KickStart():
        part.add_argument('--fsoptions', dest='fsopts')
        part.add_argument('--fstype', default='vfat',
                          choices=('ext2', 'ext3', 'ext4', 'btrfs',
-                                   'squashfs', 'vfat', 'msdos', 'swap'))
+                                   'squashfs', 'vfat', 'msdos', 'erofs',
+                                   'swap'))
        part.add_argument('--mkfs-extraopts', default='')
        part.add_argument('--label')
        part.add_argument('--use-label', action='store_true')
@@ -229,6 +230,10 @@ class KickStart():
                                err = "%s:%d: SquashFS does not support LABEL" \
                                       % (confpath, lineno)
                                raise KickStartError(err)
+                        # erofs does not support filesystem labels
+                        if parsed.fstype == 'erofs' and parsed.label:
+                            err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+                            raise KickStartError(err)
                        if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
                            if parsed.fsuuid:
                                if parsed.fsuuid.upper().startswith('0X'):
scripts/lib/wic/partition.py +13 -3
index 76d144d12d30..e0b2c5bdf29b 100644
@@ -141,9 +141,9 @@ class Partition():
                                            native_sysroot)
                self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
            else:
-                if self.fstype == 'squashfs':
-                    raise WicError("It's not possible to create empty squashfs "
-                                   "partition '%s'" % (self.mountpoint))
+                if self.fstype in ('squashfs', 'erofs'):
+                    raise WicError("It's not possible to create empty %s "
+                                   "partition '%s'" % (self.fstype, self.mountpoint))
 
                rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
                                             self.lineno, self.fstype)
@@ -369,6 +369,16 @@ class Partition():
                       (rootfs_dir, rootfs, extraopts)
        exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
 
+    def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a erofs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or ''
+        erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+                       (extraopts, self.fsuuid, rootfs, rootfs_dir)
+        exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+
    def prepare_empty_partition_ext(self, rootfs, oe_builddir,
                                    native_sysroot):
        """
scripts/lib/wic/plugins/source/rawcopy.py +3 -3
index 3c4997d8ba5e..fa7b1eb8ac75 100644
@@ -29,9 +29,9 @@ class RawCopyPlugin(SourcePlugin):
            cmd = 'btrfs filesystem label %s %s' % (dst, label)
        elif fstype == 'swap':
            cmd = 'mkswap -L %s %s' % (label, dst)
-        elif fstype == 'squashfs':
-            raise WicError("It's not possible to update a squashfs "
-                           "filesystem label '%s'" % (label))
+        elif fstype in ('squashfs', 'erofs'):
+            raise WicError("It's not possible to update a %s "
+                           "filesystem label '%s'" % (fstype, label))
        else:
            raise WicError("Cannot update filesystem label: "
                           "Unknown fstype: '%s'" % (fstype))
Signature · 1 line
2.26.2
Signature · 1 line
Sent from my Android device with K-9 Mail. Please excuse my brevity.
#7 lore raw

Quentin,

----- Ursprüngliche Mail -----

Quoted 9 lines

Von: "Quentin Schulz" <[email protected]>

meta/classes/image_types.bbclass IMAGE_TYPES = " \ · line 272

Quoted 6 lines

IMAGE_TYPES = " \

meta/classes/image_types.bbclass +1 -0
@@ -261,6 +269,7 @@ IMAGE_TYPES = " \
    wic wic.gz wic.bz2 wic.lzma wic.zst \
    container \
    f2fs \
+    erofs \

I think you're supposed to have erofs erofs-lz4 erofs-lz4hc here.

Oh yes. You're right.

Quoted 11 lines

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb

Quoted 9 lines

"

# Compression is a special case of conversion. The old variable

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +0 -0
new file mode 100644
index 000000000000..6d35d3f2fa6d

Any reason for using _git over _1.2.1?

TBH, I used meta/recipes-devtools/squashfs-tools/squashfs-tools_git.bb as template. :-) It fetches also via git and sets PV.

Quoted 5 lines

meta/recipes-devtools/erofs-utils/erofs-utils_git.bb line 2

Quoted 3 lines
meta/recipes-devtools/erofs-utils/erofs-utils_git.bb new file +2 -0
@@ -0,0 +1,18 @@
+SUMMARY = "Tools for erofs filesystems"
+LICENSE = "GPLv2"

GPL-2.0-or-later actually.

Will fix!

Thanks,
//richard