add support for zstd
Subsystems (3)
PATCH 1/1 add support for zstd
Date: Tue, 1 Jun 2021 15:43:00 +0800
Subject: [PATCH] add support for zstd
Add support for zstd
--- Makefile | 5 +++++ diskdump_mod.h | 1 + makedumpfile.c | 36 ++++++++++++++++++++++++++++++------ makedumpfile.h | 1 + 4 files changed, 37 insertions(+), 6 deletions(-)
Makefile +5 -0
@@ -68,6 +68,11 @@ endif CFLAGS += -DUSESNAPPY endif +ifeq ($(USEZSTD), on) +LIBS := -lzstd $(LIBS) +CFLAGS += -DUSEZSTD +endif + LIBS := $(LIBS) -lpthread try-run = $(shell set -e; \
diskdump_mod.h +1 -0
@@ -98,6 +98,7 @@ struct kdump_sub_header {#define DUMP_DH_COMPRESSED_INCOMPLETE 0x8/* indicate an incomplete dumpfile */ #define DUMP_DH_EXCLUDED_VMEMMAP 0x10/* unused vmemmap pages are excluded */ +#define DUMP_DH_COMPRESSED_ZSTD 0x20/* descriptor of each page for vmcore */ typedef struct page_desc {
makedumpfile.c +30 -6
@@ -26,6 +26,9 @@#include <limits.h> #include <assert.h> #include <zlib.h> +#ifdef USEZSTD +#include <zstd.h> +#endif struct symbol_table symbol_table;struct size_table size_table;
@@ -296,10 +299,10 @@ is_cache_page(unsigned long flags)static inline unsigned long calculate_len_buf_out (long page_size) { -unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy; +unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy, len_buf_out_zstd;unsigned long len_buf_out; - len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy =0 ; + len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = len_buf_out_zstd =0 ;#ifdef USELZO len_buf_out_lzo = page_size + page_size /16 +64 +3 ;
@@ -309,11 +312,14 @@ calculate_len_buf_out(long page_size) len_buf_out_snappy =snappy_max_compressed_length (page_size);#endif +#ifdef USEZSTD + len_buf_out_zstd =ZSTD_compressBound (page_size); +#endif + len_buf_out_zlib =compressBound (page_size); - len_buf_out =MAX (len_buf_out_zlib, -MAX (len_buf_out_lzo, - len_buf_out_snappy)); + len_buf_out =MAX (MAX (len_buf_out_zlib,len_buf_out_zstd), +MAX (len_buf_out_lzo,len_buf_out_snappy));return len_buf_out; }
@@ -7235,6 +7241,10 @@ write_kdump_header(void)if (info->flag_compress &DUMP_DH_COMPRESSED_ZLIB ) dh->status |=DUMP_DH_COMPRESSED_ZLIB ; +#ifdef USEZSTD + else if (info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + dh->status |= DUMP_DH_COMPRESSED_ZSTD; +#endif #ifdef USELZO else if (info->flag_compress & DUMP_DH_COMPRESSED_LZO) dh->status |= DUMP_DH_COMPRESSED_LZO;
@@ -8589,6 +8599,17 @@ write_kdump_pages_cyclic(struct cache_data *cd_header, struct cache_data *cd_pag && (size_out < info->page_size)) { pd.flags =DUMP_DH_COMPRESSED_ZLIB ; pd.size = size_out; +#ifdef USEZSTD + } else if ((info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + && (size_out = len_buf_out) + && (len_buf_out = ZSTD_compress((void *)buf_out, + size_out, (const void *)buf, info->page_size, 1)) + && (len_buf_out < info->page_size) + ) { +CHECK_ZSTD (len_buf_out); + pd.flags =DUMP_DH_COMPRESSED_ZSTD ; + pd.size = len_buf_out; +#endif #ifdef USELZO } else if (info->flag_lzo_support && (info->flag_compress & DUMP_DH_COMPRESSED_LZO)
@@ -11605,7 +11626,7 @@ main(int argc, char *argv[]) info->block_order =DEFAULT_ORDER ; message_level =DEFAULT_MSG_LEVEL ; -while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:" , longopts, +while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:z" , longopts,NULL )) !=-1 ) {switch (opt) {case OPT_BLOCK_ORDER :
@@ -11617,6 +11638,9 @@ main(int argc, char *argv[])case OPT_COMPRESS_ZLIB : info->flag_compress =DUMP_DH_COMPRESSED_ZLIB ;break ; +case OPT_COMPRESS_ZSTD : + info->flag_compress =DUMP_DH_COMPRESSED_ZSTD ; +break ;case OPT_DEBUG : flag_debug = TRUE;break ;
makedumpfile.h +1 -0
@@ -2463,6 +2463,7 @@ struct elf_prstatus {#define OPT_VERSION 'v'#define OPT_EXCLUDE_XEN_DOM 'X'#define OPT_VMLINUX 'x' +#define OPT_COMPRESS_ZSTD 'z'#define OPT_START 256#define OPT_SPLIT OPT_START+0#define OPT_REASSEMBLE OPT_START+1
Signature · 1 line
1.8.3.1
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
-----Original Message-----
Quoted 4 lines
Date: Tue, 1 Jun 2021 15:43:00 +0800
Subject: [PATCH] add support for zstdAdd support for zstd
Thanks for the patch, but we've worked on the zstd support based on these test patches:
https://github.com/k-hagio/makedumpfile/commit/f6eaf2424f41d8e9af40f7ce67e0f52b387311af
https://github.com/k-hagio/crash/commit/5c55c380120d7b7d450e07c4605aaf330f6219dc
I would like to proceed with those, please wait for a while.
Coiby, do you have any patch you can share and how is the state? IIRC, your draft patches supported also decompression etc.
Thanks,
Kazu
Quoted 143 lines
Signed-off-by Zhang Yunkai <[email protected]>--- Makefile | 5 +++++ diskdump_mod.h | 1 + makedumpfile.c | 36 ++++++++++++++++++++++++++++++------ makedumpfile.h | 1 + 4 files changed, 37 insertions(+), 6 deletions(-)Makefile +5
-0@@ -68,6 +68,11 @@ endif CFLAGS += -DUSESNAPPY endif +ifeq ($(USEZSTD), on) +LIBS := -lzstd $(LIBS) +CFLAGS += -DUSEZSTD +endif + LIBS := $(LIBS) -lpthread try-run = $(shell set -e; \diskdump_mod.h +1
-0@@ -98,6 +98,7 @@ struct kdump_sub_header {#define DUMP_DH_COMPRESSED_INCOMPLETE 0x8/* indicate an incomplete dumpfile */ #define DUMP_DH_EXCLUDED_VMEMMAP 0x10/* unused vmemmap pages are excluded */ +#define DUMP_DH_COMPRESSED_ZSTD 0x20/* descriptor of each page for vmcore */ typedef struct page_desc {makedumpfile.c +30
-6@@ -26,6 +26,9 @@#include <limits.h> #include <assert.h> #include <zlib.h> +#ifdef USEZSTD +#include <zstd.h> +#endif struct symbol_table symbol_table;struct size_table size_table;@@ -296,10 +299,10 @@ is_cache_page(unsigned long flags)static inline unsigned long calculate_len_buf_out (long page_size) { -unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy; +unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy, len_buf_out_zstd;unsigned long len_buf_out; - len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy =0 ; + len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = len_buf_out_zstd =0 ;#ifdef USELZO len_buf_out_lzo = page_size + page_size /16 +64 +3 ;@@ -309,11 +312,14 @@ calculate_len_buf_out(long page_size) len_buf_out_snappy =snappy_max_compressed_length (page_size);#endif +#ifdef USEZSTD + len_buf_out_zstd =ZSTD_compressBound (page_size); +#endif + len_buf_out_zlib =compressBound (page_size); - len_buf_out =MAX (len_buf_out_zlib, -MAX (len_buf_out_lzo, - len_buf_out_snappy)); + len_buf_out =MAX (MAX (len_buf_out_zlib,len_buf_out_zstd), +MAX (len_buf_out_lzo,len_buf_out_snappy));return len_buf_out; }@@ -7235,6 +7241,10 @@ write_kdump_header(void)if (info->flag_compress &DUMP_DH_COMPRESSED_ZLIB ) dh->status |=DUMP_DH_COMPRESSED_ZLIB ; +#ifdef USEZSTD + else if (info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + dh->status |= DUMP_DH_COMPRESSED_ZSTD; +#endif #ifdef USELZO else if (info->flag_compress & DUMP_DH_COMPRESSED_LZO) dh->status |= DUMP_DH_COMPRESSED_LZO;@@ -8589,6 +8599,17 @@ write_kdump_pages_cyclic(struct cache_data *cd_header, struct cache_data *cd_pag && (size_out < info->page_size)) { pd.flags =DUMP_DH_COMPRESSED_ZLIB ; pd.size = size_out; +#ifdef USEZSTD + } else if ((info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + && (size_out = len_buf_out) + && (len_buf_out = ZSTD_compress((void *)buf_out, + size_out, (const void *)buf, info->page_size, 1)) + && (len_buf_out < info->page_size) + ) { +CHECK_ZSTD (len_buf_out); + pd.flags =DUMP_DH_COMPRESSED_ZSTD ; + pd.size = len_buf_out; +#endif #ifdef USELZO } else if (info->flag_lzo_support && (info->flag_compress & DUMP_DH_COMPRESSED_LZO)@@ -11605,7 +11626,7 @@ main(int argc, char *argv[]) info->block_order =DEFAULT_ORDER ; message_level =DEFAULT_MSG_LEVEL ; -while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:" , longopts, +while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:z" , longopts,NULL )) !=-1 ) {switch (opt) {case OPT_BLOCK_ORDER :@@ -11617,6 +11638,9 @@ main(int argc, char *argv[])case OPT_COMPRESS_ZLIB : info->flag_compress =DUMP_DH_COMPRESSED_ZLIB ;break ; +case OPT_COMPRESS_ZSTD : + info->flag_compress =DUMP_DH_COMPRESSED_ZSTD ; +break ;case OPT_DEBUG : flag_debug = TRUE;break ;makedumpfile.h +1
-1@@ -2463,6 +2463,7 @@ struct elf_prstatus {#define OPT_VERSION 'v'#define OPT_EXCLUDE_XEN_DOM 'X'#define OPT_VMLINUX 'x' +#define OPT_COMPRESS_ZSTD 'z'#define OPT_START 256#define OPT_SPLIT OPT_START+0#define OPT_REASSEMBLE OPT_START+1 --
Quoted 1 line
1.8.3.1
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Quoted 11 lines · HAGIO KAZUHITO(萩尾 一仁)
-----Original Message-----
Quoted 4 lines
Date: Tue, 1 Jun 2021 15:43:00 +0800
Subject: [PATCH] add support for zstdAdd support for zstd
Thanks for the patch, but we've worked on the zstd support based on these test patches:
https://github.com/k-hagio/makedumpfile/commit/f6eaf2424f41d8e9af40f7ce67e0f52b387311af
https://github.com/k-hagio/crash/commit/5c55c380120d7b7d450e07c4605aaf330f6219dc
Quoted 4 lines
I would like to proceed with those, please wait for a while.
Coiby, do you have any patch you can share and how is the state? IIRC, your draft patches supported also decompression etc.
The latest work can be found on
https://github.com/coiby/makedumpfile/tree/zstd_context. I've done some simple tests following your previous suggestions. But I still need to do some tests on some huge vmcores to provide a benchmark as suggested by Dave.
Quoted 148 lines
Thanks,
KazuQuoted 144 lines
Signed-off-by Zhang Yunkai <[email protected]>--- Makefile | 5 +++++ diskdump_mod.h | 1 + makedumpfile.c | 36 ++++++++++++++++++++++++++++++------ makedumpfile.h | 1 + 4 files changed, 37 insertions(+), 6 deletions(-)Makefile +5
-0@@ -68,6 +68,11 @@ endif CFLAGS += -DUSESNAPPY endif +ifeq ($(USEZSTD), on) +LIBS := -lzstd $(LIBS) +CFLAGS += -DUSEZSTD +endif + LIBS := $(LIBS) -lpthread try-run = $(shell set -e; \diskdump_mod.h +1
-0@@ -98,6 +98,7 @@ struct kdump_sub_header {#define DUMP_DH_COMPRESSED_INCOMPLETE 0x8/* indicate an incomplete dumpfile */ #define DUMP_DH_EXCLUDED_VMEMMAP 0x10/* unused vmemmap pages are excluded */ +#define DUMP_DH_COMPRESSED_ZSTD 0x20/* descriptor of each page for vmcore */ typedef struct page_desc {makedumpfile.c +30
-6@@ -26,6 +26,9 @@#include <limits.h> #include <assert.h> #include <zlib.h> +#ifdef USEZSTD +#include <zstd.h> +#endif struct symbol_table symbol_table;struct size_table size_table;@@ -296,10 +299,10 @@ is_cache_page(unsigned long flags)static inline unsigned long calculate_len_buf_out (long page_size) { -unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy; +unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy, len_buf_out_zstd;unsigned long len_buf_out; - len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy =0 ; + len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = len_buf_out_zstd =0 ;#ifdef USELZO len_buf_out_lzo = page_size + page_size /16 +64 +3 ;@@ -309,11 +312,14 @@ calculate_len_buf_out(long page_size) len_buf_out_snappy =snappy_max_compressed_length (page_size);#endif +#ifdef USEZSTD + len_buf_out_zstd =ZSTD_compressBound (page_size); +#endif + len_buf_out_zlib =compressBound (page_size); - len_buf_out =MAX (len_buf_out_zlib, -MAX (len_buf_out_lzo, - len_buf_out_snappy)); + len_buf_out =MAX (MAX (len_buf_out_zlib,len_buf_out_zstd), +MAX (len_buf_out_lzo,len_buf_out_snappy));return len_buf_out; }@@ -7235,6 +7241,10 @@ write_kdump_header(void)if (info->flag_compress &DUMP_DH_COMPRESSED_ZLIB ) dh->status |=DUMP_DH_COMPRESSED_ZLIB ; +#ifdef USEZSTD + else if (info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + dh->status |= DUMP_DH_COMPRESSED_ZSTD; +#endif #ifdef USELZO else if (info->flag_compress & DUMP_DH_COMPRESSED_LZO) dh->status |= DUMP_DH_COMPRESSED_LZO;@@ -8589,6 +8599,17 @@ write_kdump_pages_cyclic(struct cache_data *cd_header, struct cache_data *cd_pag && (size_out < info->page_size)) { pd.flags =DUMP_DH_COMPRESSED_ZLIB ; pd.size = size_out; +#ifdef USEZSTD + } else if ((info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + && (size_out = len_buf_out) + && (len_buf_out = ZSTD_compress((void *)buf_out, + size_out, (const void *)buf, info->page_size, 1)) + && (len_buf_out < info->page_size) + ) { +CHECK_ZSTD (len_buf_out); + pd.flags =DUMP_DH_COMPRESSED_ZSTD ; + pd.size = len_buf_out; +#endif #ifdef USELZO } else if (info->flag_lzo_support && (info->flag_compress & DUMP_DH_COMPRESSED_LZO)@@ -11605,7 +11626,7 @@ main(int argc, char *argv[]) info->block_order =DEFAULT_ORDER ; message_level =DEFAULT_MSG_LEVEL ; -while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:" , longopts, +while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:z" , longopts,NULL )) !=-1 ) {switch (opt) {case OPT_BLOCK_ORDER :@@ -11617,6 +11638,9 @@ main(int argc, char *argv[])case OPT_COMPRESS_ZLIB : info->flag_compress =DUMP_DH_COMPRESSED_ZLIB ;break ; +case OPT_COMPRESS_ZSTD : + info->flag_compress =DUMP_DH_COMPRESSED_ZSTD ; +break ;case OPT_DEBUG : flag_debug = TRUE;break ;makedumpfile.h +1
-1@@ -2463,6 +2463,7 @@ struct elf_prstatus {#define OPT_VERSION 'v'#define OPT_EXCLUDE_XEN_DOM 'X'#define OPT_VMLINUX 'x' +#define OPT_COMPRESS_ZSTD 'z'#define OPT_START 256#define OPT_SPLIT OPT_START+0#define OPT_REASSEMBLE OPT_START+1 --1.8.3.1
Signature · 8 lines
Best regards, Coiby _______________________________________________ kexec mailing list [email protected] http://lists.infradead.org/mailman/listinfo/kexec
Add Coiby and Kairui since they also did some experiments on this.
Quoted 148 lines · On Sun, 6 Jun 2021 at 15:07, <[email protected]> wrote:
Date: Tue, 1 Jun 2021 15:43:00 +0800
Subject: [PATCH] add support for zstdAdd support for zstd
Signed-off-by Zhang Yunkai <[email protected]>--- Makefile | 5 +++++ diskdump_mod.h | 1 + makedumpfile.c | 36 ++++++++++++++++++++++++++++++------ makedumpfile.h | 1 + 4 files changed, 37 insertions(+), 6 deletions(-)Makefile +5
-0@@ -68,6 +68,11 @@ endif CFLAGS += -DUSESNAPPY endif +ifeq ($(USEZSTD), on) +LIBS := -lzstd $(LIBS) +CFLAGS += -DUSEZSTD +endif + LIBS := $(LIBS) -lpthread try-run = $(shell set -e; \diskdump_mod.h +1
-0@@ -98,6 +98,7 @@ struct kdump_sub_header {#define DUMP_DH_COMPRESSED_INCOMPLETE 0x8/* indicate an incomplete dumpfile */ #define DUMP_DH_EXCLUDED_VMEMMAP 0x10/* unused vmemmap pages are excluded */ +#define DUMP_DH_COMPRESSED_ZSTD 0x20/* descriptor of each page for vmcore */ typedef struct page_desc {makedumpfile.c +30
-6@@ -26,6 +26,9 @@#include <limits.h> #include <assert.h> #include <zlib.h> +#ifdef USEZSTD +#include <zstd.h> +#endif struct symbol_table symbol_table;struct size_table size_table;@@ -296,10 +299,10 @@ is_cache_page(unsigned long flags)static inline unsigned long calculate_len_buf_out (long page_size) { -unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy; +unsigned long len_buf_out_zlib, len_buf_out_lzo, len_buf_out_snappy, len_buf_out_zstd;unsigned long len_buf_out; - len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy =0 ; + len_buf_out_zlib = len_buf_out_lzo = len_buf_out_snappy = len_buf_out_zstd =0 ;#ifdef USELZO len_buf_out_lzo = page_size + page_size /16 +64 +3 ;@@ -309,11 +312,14 @@ calculate_len_buf_out(long page_size) len_buf_out_snappy =snappy_max_compressed_length (page_size);#endif +#ifdef USEZSTD + len_buf_out_zstd =ZSTD_compressBound (page_size); +#endif + len_buf_out_zlib =compressBound (page_size); - len_buf_out =MAX (len_buf_out_zlib, -MAX (len_buf_out_lzo, - len_buf_out_snappy)); + len_buf_out =MAX (MAX (len_buf_out_zlib,len_buf_out_zstd), +MAX (len_buf_out_lzo,len_buf_out_snappy));return len_buf_out; }@@ -7235,6 +7241,10 @@ write_kdump_header(void)if (info->flag_compress &DUMP_DH_COMPRESSED_ZLIB ) dh->status |=DUMP_DH_COMPRESSED_ZLIB ; +#ifdef USEZSTD + else if (info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + dh->status |= DUMP_DH_COMPRESSED_ZSTD; +#endif #ifdef USELZO else if (info->flag_compress & DUMP_DH_COMPRESSED_LZO) dh->status |= DUMP_DH_COMPRESSED_LZO;@@ -8589,6 +8599,17 @@ write_kdump_pages_cyclic(struct cache_data *cd_header, struct cache_data *cd_pag && (size_out < info->page_size)) { pd.flags =DUMP_DH_COMPRESSED_ZLIB ; pd.size = size_out; +#ifdef USEZSTD + } else if ((info->flag_compress & DUMP_DH_COMPRESSED_ZSTD) + && (size_out = len_buf_out) + && (len_buf_out = ZSTD_compress((void *)buf_out, + size_out, (const void *)buf, info->page_size, 1)) + && (len_buf_out < info->page_size) + ) { +CHECK_ZSTD (len_buf_out); + pd.flags =DUMP_DH_COMPRESSED_ZSTD ; + pd.size = len_buf_out; +#endif #ifdef USELZO } else if (info->flag_lzo_support && (info->flag_compress & DUMP_DH_COMPRESSED_LZO)@@ -11605,7 +11626,7 @@ main(int argc, char *argv[]) info->block_order =DEFAULT_ORDER ; message_level =DEFAULT_MSG_LEVEL ; -while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:" , longopts, +while ((opt =getopt_long (argc, argv,"b:cDd:eEFfg:hi:lpRvXx:z" , longopts,NULL )) !=-1 ) {switch (opt) {case OPT_BLOCK_ORDER :@@ -11617,6 +11638,9 @@ main(int argc, char *argv[])case OPT_COMPRESS_ZLIB : info->flag_compress =DUMP_DH_COMPRESSED_ZLIB ;break ; +case OPT_COMPRESS_ZSTD : + info->flag_compress =DUMP_DH_COMPRESSED_ZSTD ; +break ;case OPT_DEBUG : flag_debug = TRUE;break ;makedumpfile.h +1
-1@@ -2463,6 +2463,7 @@ struct elf_prstatus {#define OPT_VERSION 'v'#define OPT_EXCLUDE_XEN_DOM 'X'#define OPT_VMLINUX 'x' +#define OPT_COMPRESS_ZSTD 'z'#define OPT_START 256#define OPT_SPLIT OPT_START+0#define OPT_REASSEMBLE OPT_START+1 --
Quoted 4 lines
1.8.3.1_______________________________________________ kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec