Skip to contentSkip to conversation

bitbake: Add piping compression library

PATCH Superseded Joshua Watt 3 participants 6 messages Jun 17, 2021 – Jul 20, 2021 Zstd

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 bitbake: Add piping compression library

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

---
 bitbake/lib/bb/_pipecompress.py | 170 ++++++++++++++++++++++++++++++++
 bitbake/lib/bb/lz4.py           |  17 ++++
 bitbake/lib/bb/zstd.py          |  27 +++++
 3 files changed, 214 insertions(+)
 create mode 100644 bitbake/lib/bb/_pipecompress.py
 create mode 100644 bitbake/lib/bb/lz4.py
 create mode 100644 bitbake/lib/bb/zstd.py
bitbake/lib/bb/_pipecompress.py new file +170 -0
new file mode 100644
index 0000000000..3404be98d6
@@ -0,0 +1,170 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+    cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+    """
+    Open a compressed file in binary or text mode.
+
+    Users should not call this directly. A specific compression library can use
+    this helper to provide it's own "open" command
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+    "rb".
+
+    For binary mode, this function is equivalent to the cls constructor:
+    cls(filename, mode). In this case, the encoding, errors and newline
+    arguments must not be provided.
+
+    For text mode, a cls object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    file_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes, os.PathLike)):
+        binary_file = cls(filename, file_mode, **kwargs)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(
+            binary_file, encoding, errors, newline, write_through=True
+        )
+    else:
+        return binary_file
+
+
+class PipeFile(io.RawIOBase):
+    """
+    Class that implements generically piping to/from a compression program
+
+    Derived classes should add the function get_compress() and get_decompress()
+    that return the required commands. Input will be piped into stdin and the
+    (de)compressed output should be written to stdout, e.g.:
+
+        class FooFile(PipeCompressionFile):
+            def get_decompress(self):
+                return ["fooc", "--decompress", "--stdout"]
+
+            def get_compress(self):
+                return ["fooc", "--compress", "--stdout"]
+
+    """
+
+    READ = 0
+    WRITE = 1
+
+    def __init__(self, filename=None, mode="rb", *, fileobj=None):
+        if "t" in mode or "U" in mode:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if not "b" in mode:
+            mode += "b"
+
+        if mode.startswith("r"):
+            self.mode = self.READ
+        elif mode.startswith("w"):
+            self.mode = self.WRITE
+        else:
+            raise ValueError("Invalid mode %r" % mode)
+
+        if fileobj is not None:
+            self.fileobj = fileobj
+        else:
+            self.fileobj = builtins.open(filename, mode or "rb")
+
+        if self.mode == self.READ:
+            self.p = subprocess.Popen(
+                self.get_decompress(),
+                stdin=self.fileobj,
+                stdout=subprocess.PIPE,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdout
+        else:
+            self.p = subprocess.Popen(
+                self.get_compress(),
+                stdin=subprocess.PIPE,
+                stdout=self.fileobj,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdin
+
+        self.__closed = False
+
+    def close(self):
+        if self.closed:
+            return
+
+        self.pipe.close()
+        self.p.wait()
+        self.fileobj.close()
+
+        self.__closed = True
+
+    @property
+    def closed(self):
+        return self.__closed
+
+    def fileno(self):
+        return self.pipe.fileno()
+
+    def flush(self):
+        self.pipe.flush()
+
+    def isatty(self):
+        return self.pipe.isatty()
+
+    def readable(self):
+        return self.mode == self.READ
+
+    def writable(self):
+        return self.mode == self.WRITE
+
+    def readinto(self, b):
+        if self.mode != self.READ:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+            )
+        return self.pipe.readinto(b)
+
+    def write(self, data):
+        if self.mode != self.WRITE:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+            )
+        return self.pipe.write(data)
bitbake/lib/bb/lz4.py new file +17 -0
new file mode 100644
index 0000000000..c2bda291d3
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]
bitbake/lib/bb/zstd.py new file +27 -0
new file mode 100644
index 0000000000..1f46087ec9
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, **kwargs):
+        self.num_threads = num_threads
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c"]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.31.1
#1 lore raw

PATCH v2 1/1 bitbake: Add piping compression library

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

---
V2: Now with tests!

 bitbake/bin/bitbake-selftest        |   1 +
 bitbake/lib/bb/_pipecompress.py     | 194 ++++++++++++++++++++++++++++
 bitbake/lib/bb/lz4.py               |  17 +++
 bitbake/lib/bb/tests/compression.py |  82 ++++++++++++
 bitbake/lib/bb/zstd.py              |  27 ++++
 5 files changed, 321 insertions(+)
 create mode 100644 bitbake/lib/bb/_pipecompress.py
 create mode 100644 bitbake/lib/bb/lz4.py
 create mode 100644 bitbake/lib/bb/tests/compression.py
 create mode 100644 bitbake/lib/bb/zstd.py
bitbake/bin/bitbake-selftest +1 -0
index 757441c754..63762fe794 100755
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
+         "bb.tests.compression",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +194 -0

194 lines not on this page · raw message

bitbake/lib/bb/lz4.py new file +17 -0
new file mode 100644
index 0000000000..c2bda291d3
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]
bitbake/lib/bb/tests/compression.py new file +82 -0

82 lines not on this page · raw message

bitbake/lib/bb/zstd.py new file +27 -0
new file mode 100644
index 0000000000..1f46087ec9
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, **kwargs):
+        self.num_threads = num_threads
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c"]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.31.1
#2 lore raw

Hello Joshua,

Quoted 5 lines · Joshua Watt

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

Quoted 8 lines

---
V2: Now with tests!

 bitbake/bin/bitbake-selftest        |   1 +
 bitbake/lib/bb/_pipecompress.py     | 194 ++++++++++++++++++++++++++++
 bitbake/lib/bb/lz4.py               |  17 +++
 bitbake/lib/bb/tests/compression.py |  82 ++++++++++++
 bitbake/lib/bb/zstd.py              |  27 ++++

Note that this does apply on the poky git repository but not the bitbake repo unless patch -p2 is used.

Quoted 6 lines
 5 files changed, 321 insertions(+)
 create mode 100644 bitbake/lib/bb/_pipecompress.py
 create mode 100644 bitbake/lib/bb/lz4.py
 create mode 100644 bitbake/lib/bb/tests/compression.py
 create mode 100644 bitbake/lib/bb/zstd.py

I believe I have applied the patch correctly and this fails on the autobuilders with:

FileNotFoundError: [Errno 2] No such file or directory: 'zstd': 'zstd'

https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/2234
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2210
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/2213
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/2248

Quoted 359 lines
bitbake/bin/bitbake-selftest +1 -0
index 757441c754..63762fe794 100755
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
+         "bb.tests.compression",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +194 -0
new file mode 100644
index 0000000000..4b9f662143
@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+    cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+    """
+    Open a compressed file in binary or text mode.
+
+    Users should not call this directly. A specific compression library can use
+    this helper to provide it's own "open" command
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+    "rb".
+
+    For binary mode, this function is equivalent to the cls constructor:
+    cls(filename, mode). In this case, the encoding, errors and newline
+    arguments must not be provided.
+
+    For text mode, a cls object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    file_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes, os.PathLike)):
+        binary_file = cls(filename, file_mode, **kwargs)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(
+            binary_file, encoding, errors, newline, write_through=True
+        )
+    else:
+        return binary_file
+
+
+class CompressionError(OSError):
+    pass
+
+
+class PipeFile(io.RawIOBase):
+    """
+    Class that implements generically piping to/from a compression program
+
+    Derived classes should add the function get_compress() and get_decompress()
+    that return the required commands. Input will be piped into stdin and the
+    (de)compressed output should be written to stdout, e.g.:
+
+        class FooFile(PipeCompressionFile):
+            def get_decompress(self):
+                return ["fooc", "--decompress", "--stdout"]
+
+            def get_compress(self):
+                return ["fooc", "--compress", "--stdout"]
+
+    """
+
+    READ = 0
+    WRITE = 1
+
+    def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+        if "t" in mode or "U" in mode:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if not "b" in mode:
+            mode += "b"
+
+        if mode.startswith("r"):
+            self.mode = self.READ
+        elif mode.startswith("w"):
+            self.mode = self.WRITE
+        else:
+            raise ValueError("Invalid mode %r" % mode)
+
+        if fileobj is not None:
+            self.fileobj = fileobj
+        else:
+            self.fileobj = builtins.open(filename, mode or "rb")
+
+        if self.mode == self.READ:
+            self.p = subprocess.Popen(
+                self.get_decompress(),
+                stdin=self.fileobj,
+                stdout=subprocess.PIPE,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdout
+        else:
+            self.p = subprocess.Popen(
+                self.get_compress(),
+                stdin=subprocess.PIPE,
+                stdout=self.fileobj,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdin
+
+        self.__closed = False
+
+    def _check_process(self):
+        if self.p is None:
+            return
+
+        returncode = self.p.wait()
+        if returncode:
+            raise CompressionError("Process died with %d" % returncode)
+        self.p = None
+
+    def close(self):
+        if self.closed:
+            return
+
+        self.pipe.close()
+        if self.p is not None:
+            self._check_process()
+        self.fileobj.close()
+
+        self.__closed = True
+
+    @property
+    def closed(self):
+        return self.__closed
+
+    def fileno(self):
+        return self.pipe.fileno()
+
+    def flush(self):
+        self.pipe.flush()
+
+    def isatty(self):
+        return self.pipe.isatty()
+
+    def readable(self):
+        return self.mode == self.READ
+
+    def writable(self):
+        return self.mode == self.WRITE
+
+    def readinto(self, b):
+        if self.mode != self.READ:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+            )
+        size = self.pipe.readinto(b)
+        if size == 0:
+            self._check_process()
+        return size
+
+    def write(self, data):
+        if self.mode != self.WRITE:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+            )
+        data = self.pipe.write(data)
+
+        if not data:
+            self._check_process()
+
+        return data
bitbake/lib/bb/lz4.py new file +17 -0
new file mode 100644
index 0000000000..c2bda291d3
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]
bitbake/lib/bb/tests/compression.py new file +82 -0
new file mode 100644
index 0000000000..f663853901
@@ -0,0 +1,82 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.lz4
+import bb.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+    def setUp(self):
+        self._t = tempfile.TemporaryDirectory()
+        self.tmpdir = Path(self._t.name)
+        self.addCleanup(self._t.cleanup)
+
+    def _file_helper(self, mode_suffix, data):
+        tmp_file = self.tmpdir / "compressed"
+
+        with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+            f.write(data)
+
+        with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+            read_data = f.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_file(self):
+        self._file_helper("t", "Hello")
+
+    def test_binary_file(self):
+        self._file_helper("b", "Hello".encode("utf-8"))
+
+    def _pipe_helper(self, mode_suffix, data):
+        rfd, wfd = os.pipe()
+        with open(rfd, "rb") as r, open(wfd, "wb") as w:
+            with self.do_open(r, mode="r" + mode_suffix) as decompress:
+                with self.do_open(w, mode="w" + mode_suffix) as compress:
+                    compress.write(data)
+                read_data = decompress.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_pipe(self):
+        self._pipe_helper("t", "Hello")
+
+    def test_binary_pipe(self):
+        self._pipe_helper("b", "Hello".encode("utf-8"))
+
+    def test_bad_decompress(self):
+        tmp_file = self.tmpdir / "compressed"
+        with tmp_file.open("wb") as f:
+            f.write(b"\x00")
+
+        with self.assertRaises(OSError):
+            with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+                data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+    def get_cls(self):
+        return bb.lz4.LZ4File
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.lz4.open(*args, **kwargs) as f:
+            yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+    def get_cls(self):
+        return bb.zstd.ZstdFile
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.zstd.open(*args, **kwargs) as f:
+            yield f
bitbake/lib/bb/zstd.py new file +27 -0
new file mode 100644
index 0000000000..1f46087ec9
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, **kwargs):
+        self.num_threads = num_threads
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c"]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.31.1
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
#3 lore raw

PATCH v3 1/1 bitbake: Add piping compression library

In reply to Joshua Watt

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

---
V2: Now with tests!
V3: Added compression level for zstd
Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.

 bitbake/bin/bitbake-selftest        |   1 +
 bitbake/lib/bb/_pipecompress.py     | 194 ++++++++++++++++++++++++++++
 bitbake/lib/bb/lz4.py               |  17 +++
 bitbake/lib/bb/tests/compression.py |  98 ++++++++++++++
 bitbake/lib/bb/zstd.py              |  28 ++++
 5 files changed, 338 insertions(+)
 create mode 100644 bitbake/lib/bb/_pipecompress.py
 create mode 100644 bitbake/lib/bb/lz4.py
 create mode 100644 bitbake/lib/bb/tests/compression.py
 create mode 100644 bitbake/lib/bb/zstd.py
bitbake/bin/bitbake-selftest +1 -0
index 757441c754..63762fe794 100755
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
+         "bb.tests.compression",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +194 -0

194 lines not on this page · raw message

bitbake/lib/bb/lz4.py new file +17 -0
new file mode 100644
index 0000000000..c2bda291d3
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]
bitbake/lib/bb/tests/compression.py new file +98 -0

98 lines not on this page · raw message

bitbake/lib/bb/zstd.py new file +28 -0
new file mode 100644
index 0000000000..858fd062b0
@@ -0,0 +1,28 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs):
+        self.num_threads = num_threads
+        self.compresslevel = compresslevel
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c", "-%d" % self.compresslevel]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.32.0
#4 lore raw
Quoted 48 lines · Joshua Watt

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

---
V2: Now with tests!
V3: Added compression level for zstd
    Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.

 bitbake/bin/bitbake-selftest        |   1 +
 bitbake/lib/bb/_pipecompress.py     | 194 ++++++++++++++++++++++++++++
 bitbake/lib/bb/lz4.py               |  17 +++
 bitbake/lib/bb/tests/compression.py |  98 ++++++++++++++
 bitbake/lib/bb/zstd.py              |  28 ++++
 5 files changed, 338 insertions(+)

 create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/tests/compression.py create mode 100644 bitbake/lib/bb/zstd.py

bitbake/bin/bitbake-selftest +1 -0
index 757441c754..63762fe794 100755
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
           "bb.tests.runqueue",
           "bb.tests.siggen",
           "bb.tests.utils",
+         "bb.tests.compression",
           "hashserv.tests",
           "layerindexlib.tests.layerindexobj",
           "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +0 -0
new file mode 100644
index 0000000000..4b9f662143

bitbake/lib/bb/_pipecompress.py line 8

Quoted 8 lines
bitbake/lib/bb/_pipecompress.py new file +8 -0
@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created

Should or should not? :)

I'm fine with adding this but I think it should be under a bb.compress. namespace rather than directly in bb. I tweaked that patch in master-next to test that.

Cheers,

Richard

#5 lore raw
Quoted 59 lines · Richard Purdie
Quoted 48 lines · Joshua Watt

Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.

---
V2: Now with tests!
V3: Added compression level for zstd
    Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.

  bitbake/bin/bitbake-selftest        |   1 +
  bitbake/lib/bb/_pipecompress.py     | 194 ++++++++++++++++++++++++++++
  bitbake/lib/bb/lz4.py               |  17 +++
  bitbake/lib/bb/tests/compression.py |  98 ++++++++++++++
  bitbake/lib/bb/zstd.py              |  28 ++++
  5 files changed, 338 insertions(+)

 create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/tests/compression.py create mode 100644 bitbake/lib/bb/zstd.py

bitbake/bin/bitbake-selftest +1 -0
index 757441c754..63762fe794 100755
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
           "bb.tests.runqueue",
           "bb.tests.siggen",
           "bb.tests.utils",
+         "bb.tests.compression",
           "hashserv.tests",
           "layerindexlib.tests.layerindexobj",
           "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +0 -0
new file mode 100644
index 0000000000..4b9f662143

bitbake/lib/bb/_pipecompress.py line 8

Quoted 8 lines
bitbake/lib/bb/_pipecompress.py new file +8 -0
@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created

Should or should not? :)

Quoted 2 lines

I'm fine with adding this but I think it should be under a bb.compress. namespace rather than directly in bb. I tweaked that patch in master-next to test that.

That's fine with me. I would hope that having this in the code will lower the barrier to entry and it start getting used more frequently... I do have plans to use it once it's available.

Quoted 6 lines

Cheers,

Richard