Skip to contentSkip to conversation

image: Add support for ZSTD decompression

PATCH New Robert Marko 3 participants 13 messages Apr 25, 2020 – Jul 8, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +52 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
 	{	IH_COMP_LZMA,	"lzma",		"lzma compressed",	},
 	{	IH_COMP_LZO,	"lzo",		"lzo compressed",	},
 	{	IH_COMP_LZ4,	"lz4",		"lz4 compressed",	},
+	{	IH_COMP_ZSTD,	"zstd",		"zstd compressed",	},
 	{	-1,		"",		"",			},
 };
 
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong image_start, int type,
 		break;
 	}
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD
+	case IH_COMP_ZSTD: {
+		size_t size = unc_len;
+		ZSTD_DStream *dstream;
+		ZSTD_inBuffer in_buf;
+		ZSTD_outBuffer out_buf;
+		void *workspace;
+		size_t wsize;
+
+		wsize = ZSTD_DStreamWorkspaceBound(image_len);
+		workspace = malloc(wsize);
+		if (!workspace) {
+			debug("%s: cannot allocate workspace of size %zu\n", __func__,
+			      wsize);
+			return -1;
+		}
+
+		dstream = ZSTD_initDStream(image_len, workspace, wsize);
+		if (!dstream) {
+			printf("%s: ZSTD_initDStream failed\n", __func__);
+			return ZSTD_getErrorCode(ret);
+		}
+
+		in_buf.src = image_buf;
+		in_buf.pos = 0;
+		in_buf.size = image_len;
+
+		out_buf.dst = load_buf;
+		out_buf.pos = 0;
+		out_buf.size = size;
+
+		while (1) {
+			size_t ret;
+
+			ret = ZSTD_decompressStream(dstream, &out_buf, &in_buf);
+			if (ZSTD_isError(ret)) {
+				printf("%s: ZSTD_decompressStream error %d\n", __func__,
+				       ZSTD_getErrorCode(ret));
+				return ZSTD_getErrorCode(ret);
+			}
+
+			if (in_buf.pos >= image_len || !ret)
+				break;
+		}
+
+		image_len = out_buf.pos;
+
+		break;
+	}
+#endif /* CONFIG_ZSTD */
 	default:
 		printf("Unimplemented compression type %d\n", comp);
 		return -ENOSYS;
include/image.h +1 -0
index 3ffc0fdd68..8c79f6df9e 100644
@@ -308,6 +308,7 @@ enum {
 	IH_COMP_LZMA,			/* lzma  Compression Used	*/
 	IH_COMP_LZO,			/* lzo   Compression Used	*/
 	IH_COMP_LZ4,			/* lz4   Compression Used	*/
+	IH_COMP_ZSTD,			/* zstd   Compression Used	*/
 
 	IH_COMP_COUNT,
 };
Signature · 1 line
2.26.2
#1 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 30 lines · Robert Marko

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +2 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
 	{	IH_COMP_LZMA,	"lzma",		"lzma compressed",	},
 	{	IH_COMP_LZO,	"lzo",		"lzo compressed",	},
 	{	IH_COMP_LZ4,	"lz4",		"lz4 compressed",	},
+	{	IH_COMP_ZSTD,	"zstd",		"zstd compressed",	},
 	{	-1,		"",		"",			},
 };
 

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 4 lines
common/image.c +1 -0
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong image_start, int type,
 		break;
 	}
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200501/3ab0b783/attachment.sig>
#2 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 32 lines · Tom Rini
Quoted 29 lines · Robert Marko

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +2 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",      },
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",       },
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",       },
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",      },
      {       -1,             "",             "",                     },
 };
 
Quoted 8 lines

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 5 lines
common/image.c +1 -0
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong image_start, int type,
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Hi,
is that something that I need to do or?

Thanks

Quoted 3 lines

--
Tom

#3 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 41 lines · Robert Marko
Quoted 32 lines · Tom Rini
Quoted 29 lines · Robert Marko

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +2 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",      },
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",       },
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",       },
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",      },
      {       -1,             "",             "",                     },
 };
 
Quoted 8 lines

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 5 lines
common/image.c +1 -0
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong image_start, int type,
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Quoted 2 lines

Hi,
is that something that I need to do or?

Yes. You need to add the symbol, and then the code you're adding needs to make use of '#if CONFIG_IS_ENABLED(ZSTD)' rather than '#ifdef CONFIG_ZSTD'. Sorry for not being clear enough.

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200501/ddc2e77b/attachment.sig>
#4 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Hi,

I checked and SPL_ZSTD symbol already exists. But trying to use #if CONFIG_IS_ENABLED(ZSTD) inside of the switch case will fail with the preprocessor error:
In file included from tools/common/image.c:1:

Quoted 4 lines

./tools/../common/image.c: In function ?image_decomp?:
./tools/../common/image.c:510:22: error: missing binary operator before token "("
510 | #if CONFIG_IS_ENABLED(ZSTD)

Outside of the switch_case it works fine

Quoted 60 lines · Tom Rini
Quoted 10 lines · Robert Marko
Quoted 9 lines · Tom Rini
Quoted 6 lines · Robert Marko

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52

+++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +1 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
Quoted 2 lines
Quoted 2 lines

common/image.c line 48

Quoted 2 lines
common/image.c +0 -0
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char *

const argv[]);

Quoted 2 lines
Quoted 2 lines

common/image.c static const table_entry_t uimage_comp[] = { · line 194

Quoted 2 lines
common/image.c +0 -0
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 195

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 196

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 197

Quoted 1 line
common/image.c +1 -0
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 198

Quoted 1 line
common/image.c +0 -0
      {       -1,             "",             "",

},

Quoted 3 lines
Quoted 3 lines

common/image.c int image_decomp(int comp, ulong load, ulong

Quoted 3 lines
common/image.c +0 -0
 };
 
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong

image_start, int type,

Quoted 9 lines
Quoted 7 lines

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 4 lines
common/image.c +1 -0
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Hi,
is that something that I need to do or?

Yes. You need to add the symbol, and then the code you're adding needs to make use of '#if CONFIG_IS_ENABLED(ZSTD)' rather than '#ifdef CONFIG_ZSTD'. Sorry for not being clear enough.

--
Tom

#5 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 6 lines · Robert Marko

Hi,

I checked and SPL_ZSTD symbol already exists. But trying to use #if CONFIG_IS_ENABLED(ZSTD) inside of the switch case will fail with the preprocessor error:
In file included from tools/common/image.c:1:

Ah right, oops.

Quoted 7 lines
Quoted 4 lines

./tools/../common/image.c: In function ?image_decomp?:
./tools/../common/image.c:510:22: error: missing binary operator before token "("
510 | #if CONFIG_IS_ENABLED(ZSTD)

Outside of the switch_case it works fine

Sounds like <linux/kconfig.h> needs an explicit #include then.

Quoted 64 lines · Tom Rini
Quoted 61 lines · Robert Marko
Quoted 10 lines · Tom Rini
Quoted 9 lines · Robert Marko
Quoted 7 lines

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52

+++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +1 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
Quoted 2 lines
Quoted 2 lines

common/image.c line 48

Quoted 2 lines
common/image.c +0 -0
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char *

const argv[]);

Quoted 2 lines
Quoted 2 lines

common/image.c static const table_entry_t uimage_comp[] = { · line 194

Quoted 2 lines
common/image.c +0 -0
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 195

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 196

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 197

Quoted 1 line
common/image.c +1 -0
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 198

Quoted 1 line
common/image.c +0 -0
      {       -1,             "",             "",

},

Quoted 3 lines
Quoted 3 lines

common/image.c int image_decomp(int comp, ulong load, ulong

Quoted 3 lines
common/image.c +0 -0
 };
 
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong

image_start, int type,

Quoted 9 lines
Quoted 7 lines

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 4 lines
common/image.c +1 -0
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Hi,
is that something that I need to do or?

Yes. You need to add the symbol, and then the code you're adding needs to make use of '#if CONFIG_IS_ENABLED(ZSTD)' rather than '#ifdef CONFIG_ZSTD'. Sorry for not being clear enough.

--
Tom

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200504/e2eea154/attachment.sig>
#6 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 16 lines · Tom Rini
Quoted 6 lines · Robert Marko

Hi,

I checked and SPL_ZSTD symbol already exists. But trying to use #if CONFIG_IS_ENABLED(ZSTD) inside of the switch case will fail with the preprocessor error:
In file included from tools/common/image.c:1:

Ah right, oops.

Quoted 6 lines
Quoted 4 lines

./tools/../common/image.c: In function ?image_decomp?:
./tools/../common/image.c:510:22: error: missing binary operator before token "("
510 | #if CONFIG_IS_ENABLED(ZSTD)

Quoted 4 lines
Quoted 1 line

Outside of the switch_case it works fine

Sounds like <linux/kconfig.h> needs an explicit #include then.

Unfortunately, it does not help.
Preprocessor throws the same error

Quoted 70 lines
Quoted 30 lines · Tom Rini
Quoted 26 lines · Robert Marko
Quoted 10 lines · Tom Rini
Quoted 9 lines · Robert Marko
Quoted 7 lines

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52

+++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +1 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

common/image.c line 48

Quoted 2 lines
common/image.c +0 -0
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char

*

Quoted 26 lines
Quoted 26 lines

const argv[]);

Quoted 2 lines
Quoted 2 lines

common/image.c static const table_entry_t uimage_comp[] = { · line 194

Quoted 2 lines
common/image.c +0 -0
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 195

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 196

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 197

Quoted 1 line
common/image.c +1 -0
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 198

Quoted 1 line
common/image.c +0 -0
      {       -1,             "",             "",

},

Quoted 3 lines
Quoted 3 lines

common/image.c int image_decomp(int comp, ulong load, ulong

Quoted 3 lines
common/image.c +0 -0
 };
 
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong

image_start, int type,

Quoted 9 lines
Quoted 7 lines

common/image.c int image_decomp(int comp, ulong load, ulong image_start, int type, · line 485

Quoted 4 lines
common/image.c +1 -0
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Hi,
is that something that I need to do or?

Quoted 7 lines
Quoted 7 lines

Yes. You need to add the symbol, and then the code you're adding needs to make use of '#if CONFIG_IS_ENABLED(ZSTD)' rather than '#ifdef CONFIG_ZSTD'. Sorry for not being clear enough.

--
Tom

--
Tom

#7 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Tom,
I have tried various things but CONFIG_IS_ENABLED won't work inside of switch case.
It works fine outside of if though.

Quoted 96 lines · Robert Marko
Quoted 17 lines · Tom Rini
Quoted 6 lines · Robert Marko

Hi,

I checked and SPL_ZSTD symbol already exists. But trying to use #if CONFIG_IS_ENABLED(ZSTD) inside of the switch case will fail with the preprocessor error:
In file included from tools/common/image.c:1:

Ah right, oops.

Quoted 6 lines
Quoted 4 lines

./tools/../common/image.c: In function ?image_decomp?:
./tools/../common/image.c:510:22: error: missing binary operator before token "("
510 | #if CONFIG_IS_ENABLED(ZSTD)

Quoted 3 lines
Quoted 1 line

Outside of the switch_case it works fine

Sounds like <linux/kconfig.h> needs an explicit #include then.

Unfortunately, it does not help.
Preprocessor throws the same error

Quoted 69 lines
Quoted 59 lines · Tom Rini
Quoted 54 lines · Robert Marko
Quoted 10 lines · Tom Rini
Quoted 9 lines · Robert Marko
Quoted 7 lines

This patch adds support for ZSTD decompression of FIT images.

---
 common/image.c  | 52

+++++++++++++++++++++++++++++++++++++++++++++++++

Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
 include/image.h |  1 +
 2 files changed, 53 insertions(+)
common/image.c +1 -0
index 94873cb6ed..70ba0f4328 100644
@@ -42,6 +42,7 @@
 #include <lzma/LzmaTypes.h>
 #include <lzma/LzmaDec.h>
 #include <lzma/LzmaTools.h>
+#include <linux/zstd.h>
 
Quoted 2 lines
Quoted 2 lines

common/image.c line 48

Quoted 2 lines
common/image.c +0 -0
 #ifdef CONFIG_CMD_BDI
 extern int do_bdinfo(cmd_tbl_t *cmdtp, int flag, int argc, char *

const argv[]);

Quoted 2 lines
Quoted 2 lines

common/image.c static const table_entry_t uimage_comp[] = { · line 194

Quoted 2 lines
common/image.c +0 -0
@@ -193,6 +194,7 @@ static const table_entry_t uimage_comp[] = {
      {       IH_COMP_LZMA,   "lzma",         "lzma compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 195

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZO,    "lzo",          "lzo compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 196

Quoted 1 line
common/image.c +0 -0
      {       IH_COMP_LZ4,    "lz4",          "lz4 compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 197

Quoted 1 line
common/image.c +1 -0
+     {       IH_COMP_ZSTD,   "zstd",         "zstd compressed",

},

Quoted 1 line
Quoted 1 line

common/image.c static const table_entry_t uimage_comp[] = { · line 198

Quoted 1 line
common/image.c +0 -0
      {       -1,             "",             "",

},

Quoted 3 lines
Quoted 3 lines

common/image.c int image_decomp(int comp, ulong load, ulong

Quoted 3 lines
common/image.c +0 -0
 };
 
@@ -480,6 +482,56 @@ int image_decomp(int comp, ulong load, ulong

image_start, int type,

Quoted 9 lines
Quoted 7 lines

common/image.c int image_decomp(int comp, ulong load, ulong · line 485

Quoted 4 lines
common/image.c +1 -0
              break;
      }
 #endif /* CONFIG_LZ4 */
+#ifdef CONFIG_ZSTD

We need to add SPL_ZSTD as a symbol to lib/Kconfig and then use CONFIG_IS_ENABLED() tests here to avoid growth in SPL. Thanks!

Hi,
is that something that I need to do or?

Quoted 7 lines
Quoted 7 lines

Yes. You need to add the symbol, and then the code you're adding needs to make use of '#if CONFIG_IS_ENABLED(ZSTD)' rather than '#ifdef CONFIG_ZSTD'. Sorry for not being clear enough.

--
Tom

--
Tom

#8 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 4 lines · Robert Marko

Tom,
I have tried various things but CONFIG_IS_ENABLED won't work inside of switch case.
It works fine outside of if though.

OK, thanks, I'll poke things more to make sure what I'm worried about doesn't happen.

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200520/66e18932/attachment.sig>
#9 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 10 lines · Tom Rini
Quoted 4 lines · Robert Marko

Tom,
I have tried various things but CONFIG_IS_ENABLED won't work inside of switch case.
It works fine outside of if though.

OK, thanks, I'll poke things more to make sure what I'm worried about doesn't happen.

Hi,
were you able to test the commit?

Regards,
Robert

Quoted 3 lines

--
Tom

#10 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 9 lines · Robert Marko
Quoted 8 lines · Tom Rini
Quoted 4 lines · Robert Marko

Tom,
I have tried various things but CONFIG_IS_ENABLED won't work inside of switch case.
It works fine outside of if though.

Quoted 5 lines
Quoted 2 lines

OK, thanks, I'll poke things more to make sure what I'm worried about doesn't happen.

Hi,
were you able to test the commit?

So I took a look at this since I'm also interested in getting zstd support in. And the problem is that when building host tools, the IS_ENABLED / CONFIG_IS_ENABLED logic doesn't make much sense anyway - one can of course include the kconfig.h header that defines those macros to avoid the cpp error (one has to be careful, because the headers included vary wildly depending on USE_HOSTCC), but I don't think it's meaningful - the host tools should not depend on configuration for any specific board.

Nor do they currently; the use of ifdef CONFIG_GZIP for example just means that that part is ignored for the host tools - doing

common/image.c +0 -0
@@ -430,12 +430,12 @@ int image_decomp(int comp, ulong load, ulong

image_start, int type,

                else
                        ret = -ENOSPC;
                break;
-#ifdef CONFIG_GZIP
+//#ifdef CONFIG_GZIP
        case IH_COMP_GZIP: {
                ret = gunzip(load_buf, unc_len, image_buf, &image_len);
                break;
        }
-#endif /* CONFIG_GZIP */
+//#endif /* CONFIG_GZIP */
 #ifdef CONFIG_BZIP2
        case IH_COMP_BZIP2: {
                uint size = unc_len;

should be a no-op when CONFIG_GZIP=y, but it breaks the build of the host tools.

That, in turn, means that the fit_check_sign tool (which AFAICT is the only host tool that actually care about this - we really should build host tools with the -ffunction-sections,-fdata-sections,-Wl,--gc-sections flags) currently "works" in the sense that it does check the signature, but it doesn't really check if the image(s) can be decompressed using the vendored copy of the decompression code; hidden in the output is

Unimplemented compression type 5

but for some reason that ends up being ok.

So, for now, I suggest simply surrounding all the CASE_IH_COMP_* (except NONE) by a #ifndef USE_HOSTCC - I don't think that will change anything at all for the existing host tools. Then you should be able to use the proper CONFIG_IS_ENABLED(ZSTD) to guard the IH_COMP_ZSTD case. Also, it would allow us to change the existing bare ifdefs to be SPL-aware, i.e. change #ifdef CONFIG_GZIP to #if CONFIG_IS_ENABLED(GZIP) [assuming the SPL_GZIP symbol actually exists].

Rasmus

#11 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

Quoted 62 lines · Rasmus Villemoes
Quoted 9 lines · Robert Marko
Quoted 8 lines · Tom Rini
Quoted 4 lines · Robert Marko

Tom,
I have tried various things but CONFIG_IS_ENABLED won't work inside of switch case.
It works fine outside of if though.

Quoted 5 lines
Quoted 2 lines

OK, thanks, I'll poke things more to make sure what I'm worried about doesn't happen.

Hi,
were you able to test the commit?

So I took a look at this since I'm also interested in getting zstd support in. And the problem is that when building host tools, the IS_ENABLED / CONFIG_IS_ENABLED logic doesn't make much sense anyway - one can of course include the kconfig.h header that defines those macros to avoid the cpp error (one has to be careful, because the headers included vary wildly depending on USE_HOSTCC), but I don't think it's meaningful - the host tools should not depend on configuration for any specific board.

Nor do they currently; the use of ifdef CONFIG_GZIP for example just means that that part is ignored for the host tools - doing

common/image.c +2 -2
@@ -430,12 +430,12 @@ int image_decomp(int comp, ulong load, ulong
 image_start, int type,
                else
                        ret = -ENOSPC;
                break;
-#ifdef CONFIG_GZIP
+//#ifdef CONFIG_GZIP
        case IH_COMP_GZIP: {
                ret = gunzip(load_buf, unc_len, image_buf, &image_len);
                break;
        }
-#endif /* CONFIG_GZIP */
+//#endif /* CONFIG_GZIP */
 #ifdef CONFIG_BZIP2
        case IH_COMP_BZIP2: {
                uint size = unc_len;

should be a no-op when CONFIG_GZIP=y, but it breaks the build of the host tools.

That, in turn, means that the fit_check_sign tool (which AFAICT is the only host tool that actually care about this - we really should build host tools with the -ffunction-sections,-fdata-sections,-Wl,--gc-sections flags) currently "works" in the sense that it does check the signature, but it doesn't really check if the image(s) can be decompressed using the vendored copy of the decompression code; hidden in the output is

Unimplemented compression type 5

but for some reason that ends up being ok.

Quoted 7 lines

So, for now, I suggest simply surrounding all the CASE_IH_COMP_* (except NONE) by a #ifndef USE_HOSTCC - I don't think that will change anything at all for the existing host tools. Then you should be able to use the proper CONFIG_IS_ENABLED(ZSTD) to guard the IH_COMP_ZSTD case. Also, it would allow us to change the existing bare ifdefs to be SPL-aware, i.e. change #ifdef CONFIG_GZIP to #if CONFIG_IS_ENABLED(GZIP) [assuming the SPL_GZIP symbol actually exists].

Alright, so first thanks for digging here. Looking over the tools, for verifying the signature we don't need to decompress the contents as we are just checking the signature. For dumpimage it's also OK to dump a compressed file and leave it to regular tools to decompress. Do you want to make a patch to guard the existing section and CONFIG_IS_ENABLED it or should I and Suggested-by you? Thanks!

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200611/e87c666b/attachment.sig>
#12 lore raw

PATCH 1/1 image: Add support for ZSTD decompression

In reply to Robert Marko
Quoted 4 lines · Robert Marko

This patch adds support for ZSTD decompression of FIT images.

Applied to u-boot/master, thanks!

Signature · 8 lines
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 659 bytes
Desc: not available
URL: <https://lists.denx.de/pipermail/u-boot/attachments/20200707/dbe07042/attachment.sig>