bitbake: Add piping compression library
PATCH 1/1 bitbake: Add piping compression library
Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
--- bitbake/lib/bb/_pipecompress.py | 170 ++++++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 ++++ bitbake/lib/bb/zstd.py | 27 +++++ 3 files changed, 214 insertions(+) create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/zstd.py
bitbake/lib/bb/_pipecompress.py new file +170 -0
@@ -0,0 +1,170 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Helper library to implement streaming compression and decompression using an +# external process +# +# This library should be used directly by end users; a wrapper library for the +# specific compression tool should be created + +import builtins +import io +import os +import subprocess + + +def open_wrap( + cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs +): + """ + Open a compressed file in binary or text mode. + + Users should not call this directly. A specific compression library can use + this helper to provide it's own "open" command + + The filename argument can be an actual filename (a str or bytes object), or + an existing file object to read from or write to. + + The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for + binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is + "rb". + + For binary mode, this function is equivalent to the cls constructor: + cls(filename, mode). In this case, the encoding, errors and newline + arguments must not be provided. + + For text mode, a cls object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error handling + behavior, and line ending(s). + """ + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + + file_mode = mode.replace("t", "") + if isinstance(filename, (str, bytes, os.PathLike)): + binary_file = cls(filename, file_mode, **kwargs) + elif hasattr(filename, "read") or hasattr(filename, "write"): + binary_file = cls(None, file_mode, fileobj=filename, **kwargs) + else: + raise TypeError("filename must be a str or bytes object, or a file") + + if "t" in mode: + return io.TextIOWrapper( + binary_file, encoding, errors, newline, write_through=True + ) + else: + return binary_file + + +class PipeFile(io.RawIOBase): + """ + Class that implements generically piping to/from a compression program + + Derived classes should add the function get_compress() and get_decompress() + that return the required commands. Input will be piped into stdin and the + (de)compressed output should be written to stdout, e.g.: + + class FooFile(PipeCompressionFile): + def get_decompress(self): + return ["fooc", "--decompress", "--stdout"] + + def get_compress(self): + return ["fooc", "--compress", "--stdout"] + + """ + + READ = 0 + WRITE = 1 + + def __init__(self, filename=None, mode="rb", *, fileobj=None): + if "t" in mode or "U" in mode: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if not "b" in mode: + mode += "b" + + if mode.startswith("r"): + self.mode = self.READ + elif mode.startswith("w"): + self.mode = self.WRITE + else: + raise ValueError("Invalid mode %r" % mode) + + if fileobj is not None: + self.fileobj = fileobj + else: + self.fileobj = builtins.open(filename, mode or "rb") + + if self.mode == self.READ: + self.p = subprocess.Popen( + self.get_decompress(), + stdin=self.fileobj, + stdout=subprocess.PIPE, + close_fds=True, + ) + self.pipe = self.p.stdout + else: + self.p = subprocess.Popen( + self.get_compress(), + stdin=subprocess.PIPE, + stdout=self.fileobj, + close_fds=True, + ) + self.pipe = self.p.stdin + + self.__closed = False + + def close(self): + if self.closed: + return + + self.pipe.close() + self.p.wait() + self.fileobj.close() + + self.__closed = True + + @property + def closed(self): + return self.__closed + + def fileno(self): + return self.pipe.fileno() + + def flush(self): + self.pipe.flush() + + def isatty(self): + return self.pipe.isatty() + + def readable(self): + return self.mode == self.READ + + def writable(self): + return self.mode == self.WRITE + + def readinto(self, b): + if self.mode != self.READ: + import errno + + raise OSError( + errno.EBADF, "read() on write-only %s object" % self.__class__.__name__ + ) + return self.pipe.readinto(b) + + def write(self, data): + if self.mode != self.WRITE: + import errno + + raise OSError( + errno.EBADF, "write() on read-only %s object" % self.__class__.__name__ + ) + return self.pipe.write(data)
bitbake/lib/bb/lz4.py new file +17 -0
@@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs) + + +class LZ4File(bb._pipecompress.PipeFile): + def get_compress(self): + return ["lz4c", "-z", "-c"] + + def get_decompress(self): + return ["lz4c", "-d", "-c"]
bitbake/lib/bb/zstd.py new file +27 -0
@@ -0,0 +1,27 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress +import shutil + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs) + + +class ZstdFile(bb._pipecompress.PipeFile): + def __init__(self, *args, num_threads=1, **kwargs): + self.num_threads = num_threads + super().__init__(*args, **kwargs) + + def _get_zstd(self): + if self.num_threads == 1 or not shutil.which("pzstd"): + return ["zstd"] + return ["pzstd", "-p", "%d" % self.num_threads] + + def get_compress(self): + return self._get_zstd() + ["-c"] + + def get_decompress(self): + return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.31.1
PATCH v2 1/1 bitbake: Add piping compression library
Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
---
V2: Now with tests!
bitbake/bin/bitbake-selftest | 1 + bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 +++ bitbake/lib/bb/tests/compression.py | 82 ++++++++++++ bitbake/lib/bb/zstd.py | 27 ++++ 5 files changed, 321 insertions(+) create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/tests/compression.py create mode 100644 bitbake/lib/bb/zstd.py
bitbake/bin/bitbake-selftest +1 -0
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser", "bb.tests.runqueue", "bb.tests.siggen", "bb.tests.utils", + "bb.tests.compression", "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +194 -0
194 lines not on this page · raw message
bitbake/lib/bb/lz4.py new file +17 -0
@@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs) + + +class LZ4File(bb._pipecompress.PipeFile): + def get_compress(self): + return ["lz4c", "-z", "-c"] + + def get_decompress(self): + return ["lz4c", "-d", "-c"]
bitbake/lib/bb/tests/compression.py new file +82 -0
82 lines not on this page · raw message
bitbake/lib/bb/zstd.py new file +27 -0
@@ -0,0 +1,27 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress +import shutil + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs) + + +class ZstdFile(bb._pipecompress.PipeFile): + def __init__(self, *args, num_threads=1, **kwargs): + self.num_threads = num_threads + super().__init__(*args, **kwargs) + + def _get_zstd(self): + if self.num_threads == 1 or not shutil.which("pzstd"): + return ["zstd"] + return ["pzstd", "-p", "%d" % self.num_threads] + + def get_compress(self): + return self._get_zstd() + ["-c"] + + def get_decompress(self): + return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.31.1
Hello Joshua,
Quoted 5 lines · Joshua Watt
Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
Signed-off-by Joshua Watt <[email protected]>
Quoted 8 lines
---
V2: Now with tests!bitbake/bin/bitbake-selftest | 1 + bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 +++ bitbake/lib/bb/tests/compression.py | 82 ++++++++++++ bitbake/lib/bb/zstd.py | 27 ++++
Note that this does apply on the poky git repository but not the bitbake repo unless patch -p2 is used.
Quoted 6 lines
5 files changed, 321 insertions(+) create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/tests/compression.py create mode 100644 bitbake/lib/bb/zstd.py
I believe I have applied the patch correctly and this fails on the autobuilders with:
FileNotFoundError: [Errno 2] No such file or directory: 'zstd': 'zstd'
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/2234
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2210
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/2213
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/2248
Quoted 359 lines
bitbake/bin/bitbake-selftest +1
-0@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser", "bb.tests.runqueue", "bb.tests.siggen", "bb.tests.utils", + "bb.tests.compression", "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi",bitbake/lib/bb/_pipecompress.py new file +194
-0@@ -0,0 +1,194 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Helper library to implement streaming compression and decompression using an +# external process +# +# This library should be used directly by end users; a wrapper library for the +# specific compression tool should be created + +import builtins +import io +import os +import subprocess + + +def open_wrap( + cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs +): + """ + Open a compressed file in binary or text mode. + + Users should not call this directly. A specific compression library can use + this helper to provide it's own "open" command + + The filename argument can be an actual filename (a str or bytes object), or + an existing file object to read from or write to. + + The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for + binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is + "rb". + + For binary mode, this function is equivalent to the cls constructor: + cls(filename, mode). In this case, the encoding, errors and newline + arguments must not be provided. + + For text mode, a cls object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error handling + behavior, and line ending(s). + """ + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + + file_mode = mode.replace("t", "") + if isinstance(filename, (str, bytes, os.PathLike)): + binary_file = cls(filename, file_mode, **kwargs) + elif hasattr(filename, "read") or hasattr(filename, "write"): + binary_file = cls(None, file_mode, fileobj=filename, **kwargs) + else: + raise TypeError("filename must be a str or bytes object, or a file") + + if "t" in mode: + return io.TextIOWrapper( + binary_file, encoding, errors, newline, write_through=True + ) + else: + return binary_file + + +class CompressionError(OSError): + pass + + +class PipeFile(io.RawIOBase): + """ + Class that implements generically piping to/from a compression program + + Derived classes should add the function get_compress() and get_decompress() + that return the required commands. Input will be piped into stdin and the + (de)compressed output should be written to stdout, e.g.: + + class FooFile(PipeCompressionFile): + def get_decompress(self): + return ["fooc", "--decompress", "--stdout"] + + def get_compress(self): + return ["fooc", "--compress", "--stdout"] + + """ + + READ = 0 + WRITE = 1 + + def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None): + if "t" in mode or "U" in mode: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if not "b" in mode: + mode += "b" + + if mode.startswith("r"): + self.mode = self.READ + elif mode.startswith("w"): + self.mode = self.WRITE + else: + raise ValueError("Invalid mode %r" % mode) + + if fileobj is not None: + self.fileobj = fileobj + else: + self.fileobj = builtins.open(filename, mode or "rb") + + if self.mode == self.READ: + self.p = subprocess.Popen( + self.get_decompress(), + stdin=self.fileobj, + stdout=subprocess.PIPE, + stderr=stderr, + close_fds=True, + ) + self.pipe = self.p.stdout + else: + self.p = subprocess.Popen( + self.get_compress(), + stdin=subprocess.PIPE, + stdout=self.fileobj, + stderr=stderr, + close_fds=True, + ) + self.pipe = self.p.stdin + + self.__closed = False + + def _check_process(self): + if self.p is None: + return + + returncode = self.p.wait() + if returncode: + raise CompressionError("Process died with %d" % returncode) + self.p = None + + def close(self): + if self.closed: + return + + self.pipe.close() + if self.p is not None: + self._check_process() + self.fileobj.close() + + self.__closed = True + + @property + def closed(self): + return self.__closed + + def fileno(self): + return self.pipe.fileno() + + def flush(self): + self.pipe.flush() + + def isatty(self): + return self.pipe.isatty() + + def readable(self): + return self.mode == self.READ + + def writable(self): + return self.mode == self.WRITE + + def readinto(self, b): + if self.mode != self.READ: + import errno + + raise OSError( + errno.EBADF, "read() on write-only %s object" % self.__class__.__name__ + ) + size = self.pipe.readinto(b) + if size == 0: + self._check_process() + return size + + def write(self, data): + if self.mode != self.WRITE: + import errno + + raise OSError( + errno.EBADF, "write() on read-only %s object" % self.__class__.__name__ + ) + data = self.pipe.write(data) + + if not data: + self._check_process() + + return databitbake/lib/bb/lz4.py new file +17
-0@@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs) + + +class LZ4File(bb._pipecompress.PipeFile): + def get_compress(self): + return ["lz4c", "-z", "-c"] + + def get_decompress(self): + return ["lz4c", "-d", "-c"]bitbake/lib/bb/tests/compression.py new file +82
-0@@ -0,0 +1,82 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from pathlib import Path +import bb.lz4 +import bb.zstd +import contextlib +import os +import shutil +import tempfile +import unittest +import subprocess + + +class CompressionTests(object): + def setUp(self): + self._t = tempfile.TemporaryDirectory() + self.tmpdir = Path(self._t.name) + self.addCleanup(self._t.cleanup) + + def _file_helper(self, mode_suffix, data): + tmp_file = self.tmpdir / "compressed" + + with self.do_open(tmp_file, mode="w" + mode_suffix) as f: + f.write(data) + + with self.do_open(tmp_file, mode="r" + mode_suffix) as f: + read_data = f.read() + + self.assertEqual(read_data, data) + + def test_text_file(self): + self._file_helper("t", "Hello") + + def test_binary_file(self): + self._file_helper("b", "Hello".encode("utf-8")) + + def _pipe_helper(self, mode_suffix, data): + rfd, wfd = os.pipe() + with open(rfd, "rb") as r, open(wfd, "wb") as w: + with self.do_open(r, mode="r" + mode_suffix) as decompress: + with self.do_open(w, mode="w" + mode_suffix) as compress: + compress.write(data) + read_data = decompress.read() + + self.assertEqual(read_data, data) + + def test_text_pipe(self): + self._pipe_helper("t", "Hello") + + def test_binary_pipe(self): + self._pipe_helper("b", "Hello".encode("utf-8")) + + def test_bad_decompress(self): + tmp_file = self.tmpdir / "compressed" + with tmp_file.open("wb") as f: + f.write(b"\x00") + + with self.assertRaises(OSError): + with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f: + data = f.read() + + +class LZ4Tests(CompressionTests, unittest.TestCase): + def get_cls(self): + return bb.lz4.LZ4File + + @contextlib.contextmanager + def do_open(self, *args, **kwargs): + with bb.lz4.open(*args, **kwargs) as f: + yield f + + +class ZStdTests(CompressionTests, unittest.TestCase): + def get_cls(self): + return bb.zstd.ZstdFile + + @contextlib.contextmanager + def do_open(self, *args, **kwargs): + with bb.zstd.open(*args, **kwargs) as f: + yield fbitbake/lib/bb/zstd.py new file +27
-0@@ -0,0 +1,27 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress +import shutil + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs) + + +class ZstdFile(bb._pipecompress.PipeFile): + def __init__(self, *args, num_threads=1, **kwargs): + self.num_threads = num_threads + super().__init__(*args, **kwargs) + + def _get_zstd(self): + if self.num_threads == 1 or not shutil.which("pzstd"): + return ["zstd"] + return ["pzstd", "-p", "%d" % self.num_threads] + + def get_compress(self): + return self._get_zstd() + ["-c"] + + def get_decompress(self): + return self._get_zstd() + ["-d", "-c"]Signature · 1 line
2.31.1
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
PATCH v3 1/1 bitbake: Add piping compression library
In reply to Joshua WattAdds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
---
V2: Now with tests!
V3: Added compression level for zstd
Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.
bitbake/bin/bitbake-selftest | 1 + bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 +++ bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++ bitbake/lib/bb/zstd.py | 28 ++++ 5 files changed, 338 insertions(+) create mode 100644 bitbake/lib/bb/_pipecompress.py create mode 100644 bitbake/lib/bb/lz4.py create mode 100644 bitbake/lib/bb/tests/compression.py create mode 100644 bitbake/lib/bb/zstd.py
bitbake/bin/bitbake-selftest +1 -0
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser", "bb.tests.runqueue", "bb.tests.siggen", "bb.tests.utils", + "bb.tests.compression", "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi",
bitbake/lib/bb/_pipecompress.py new file +194 -0
194 lines not on this page · raw message
bitbake/lib/bb/lz4.py new file +17 -0
@@ -0,0 +1,17 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs) + + +class LZ4File(bb._pipecompress.PipeFile): + def get_compress(self): + return ["lz4c", "-z", "-c"] + + def get_decompress(self): + return ["lz4c", "-d", "-c"]
bitbake/lib/bb/tests/compression.py new file +98 -0
98 lines not on this page · raw message
bitbake/lib/bb/zstd.py new file +28 -0
@@ -0,0 +1,28 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import bb._pipecompress +import shutil + + +def open(*args, **kwargs): + return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs) + + +class ZstdFile(bb._pipecompress.PipeFile): + def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs): + self.num_threads = num_threads + self.compresslevel = compresslevel + super().__init__(*args, **kwargs) + + def _get_zstd(self): + if self.num_threads == 1 or not shutil.which("pzstd"): + return ["zstd"] + return ["pzstd", "-p", "%d" % self.num_threads] + + def get_compress(self): + return self._get_zstd() + ["-c", "-%d" % self.compresslevel] + + def get_decompress(self): + return self._get_zstd() + ["-d", "-c"]
Signature · 1 line
2.32.0
Quoted 48 lines · Joshua Watt
Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
Signed-off-by Joshua Watt <[email protected]>---
V2: Now with tests!
V3: Added compression level for zstd
Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.bitbake/bin/bitbake-selftest | 1 + bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 +++ bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++ bitbake/lib/bb/zstd.py | 28 ++++ 5 files changed, 338 insertions(+)create mode 100644
bitbake/lib/bb/_pipecompress.pycreate mode 100644bitbake/lib/bb/lz4.pycreate mode 100644bitbake/lib/bb/tests/compression.pycreate mode 100644bitbake/lib/bb/zstd.pybitbake/bin/bitbake-selftest +1
-0@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser", "bb.tests.runqueue", "bb.tests.siggen", "bb.tests.utils", + "bb.tests.compression", "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi",bitbake/lib/bb/_pipecompress.py new file +0
-0
bitbake/lib/bb/_pipecompress.py line 8
Quoted 8 lines
bitbake/lib/bb/_pipecompress.py new file +8
-0@@ -0,0 +1,194 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Helper library to implement streaming compression and decompression using an +# external process +# +# This library should be used directly by end users; a wrapper library for the +# specific compression tool should be created
Should or should not? :)
I'm fine with adding this but I think it should be under a bb.compress. namespace rather than directly in bb. I tweaked that patch in master-next to test that.
Cheers,
Richard
Quoted 59 lines · Richard Purdie
Quoted 48 lines · Joshua Watt
Adds a library that implements file-like objects (similar to gzip.GzipFile) that can stream to arbitrary compression programs. This is utilized to implement a LZ4 and zstd compression API.
Signed-off-by Joshua Watt <[email protected]>---
V2: Now with tests!
V3: Added compression level for zstd
Tests are skipped if command line programs are missing, which should prevent needing to update bitbake and oe-core in lock-step.bitbake/bin/bitbake-selftest | 1 + bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++ bitbake/lib/bb/lz4.py | 17 +++ bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++ bitbake/lib/bb/zstd.py | 28 ++++ 5 files changed, 338 insertions(+)create mode 100644
bitbake/lib/bb/_pipecompress.pycreate mode 100644bitbake/lib/bb/lz4.pycreate mode 100644bitbake/lib/bb/tests/compression.pycreate mode 100644bitbake/lib/bb/zstd.pybitbake/bin/bitbake-selftest +1
-0@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser", "bb.tests.runqueue", "bb.tests.siggen", "bb.tests.utils", + "bb.tests.compression", "hashserv.tests", "layerindexlib.tests.layerindexobj", "layerindexlib.tests.restapi",bitbake/lib/bb/_pipecompress.py new file +0
-0bitbake/lib/bb/_pipecompress.py line 8
Quoted 8 lines
bitbake/lib/bb/_pipecompress.py new file +8
-0@@ -0,0 +1,194 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Helper library to implement streaming compression and decompression using an +# external process +# +# This library should be used directly by end users; a wrapper library for the +# specific compression tool should be createdShould or should not? :)
Quoted 2 lines
I'm fine with adding this but I think it should be under a bb.compress. namespace rather than directly in bb. I tweaked that patch in master-next to test that.
That's fine with me. I would hope that having this in the code will lower the barrier to entry and it start getting used more frequently... I do have plans to use it once it's available.
Quoted 6 lines
Cheers,
Richard