Use progress bar also for bitmap creation
Subsystems (2)
PATCH 1/1 Use progress bar also for bitmap creation
On large nodes, bitmap creation takes a serious amount of time. But the progress indicator only starts after the bitmaps has been created, showing only the progress of the copy process.
This patch adds a "message" parameter to the print_progress() function. That message is displayed first and gets overwritten by next message. So, the user sees
Excluding zero pages : [ 0 %]
Excluding zero pages : [... %]
Excluding zero pages : [100 %]
Copying data : [ 0 %]
Copying data : [... %]
Copying data : [100 %]
in sequence. IMO that's the fastest option to implement such a progress.
--- makedumpfile.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-)
makedumpfile.c +23 -7
@@ -32,6 +32,17 @@ int message_level;/* * Forward declarations */ +void print_progress (const char *msg, +unsigned long current, +unsigned long end); + +/* +* Message texts +*/ +#define PROGRESS_COPY "Copying data" +#define PROGRESS_UNN_PAGES "Excluding unnecessary pages" +#define PROGRESS_ZERO_PAGES "Excluding zero pages" +#define PROGRESS_MAXLEN "35"/* * The numbers of the excluded pages
@@ -4062,6 +4073,9 @@ exclude_zero_pages() }for (pfn = paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_ZERO_PAGES , pfn, info->max_mapnr); +if (!is_in_segs (paddr))continue ;
@@ -4100,6 +4114,8 @@ exclude_unnecessary_pages() goto out; }for (mm =0 ; mm < info->num_mem_map; mm++) { +print_progress (PROGRESS_UNN_PAGES , mm, info->num_mem_map); + mmd = &info->mem_map_data[mm]; pfn = mmd->pfn_start; paddr = pfn*info->page_size;
@@ -4670,7 +4686,7 @@ write_kdump_header() }void -print_progress (unsigned long current,unsigned long end) +print_progress (const char *msg,unsigned long current,unsigned long end) {int progress;time_t tm;
@@ -4686,7 +4702,7 @@ print_progress(unsigned long current, un progress =100 ;PROGRESS_MSG ("\r" ); -PROGRESS_MSG ("[%3d %%]" , progress); +PROGRESS_MSG ("%-" PROGRESS_MAXLEN "s: [%3d %%] " , msg, progress); }int
@@ -4912,7 +4928,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;
@@ -5009,7 +5025,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;
@@ -5038,7 +5054,7 @@ write_elf_pages()if (!write_cache_bufsz (&cd_seg)) goto out; -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;
@@ -5247,7 +5263,7 @@ write_kdump_pages()for (pfn =0 ; pfn < info->max_mapnr; pfn++) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if ((pfn %PFN_BUFBITMAP ) ==0 ) {if (info->len_bitmap - bm2.offset <BUFSIZE_BITMAP )
@@ -5325,7 +5341,7 @@ write_kdump_pages()/* * Print the progress of the end. */ -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Hi Bernhard,
Thank you for your patch.
I like this idea :-)
I am busy now, and I will consider the patch well the next week.
Thanks
Ken'ichi Ohmichi
Quoted 119 lines · Bernhard Walle
On large nodes, bitmap creation takes a serious amount of time. But the progress indicator only starts after the bitmaps has been created, showing only the progress of the copy process.
This patch adds a "message" parameter to the
print_progress()function. That message is displayed first and gets overwritten by next message. So, the user seesExcluding zero pages : [ 0 %]
Excluding zero pages : [... %]
Excluding zero pages : [100 %]
Copying data : [ 0 %]
Copying data : [... %]
Copying data : [100 %]in sequence. IMO that's the fastest option to implement such a progress.
Signed-off-by Bernhard Walle <[email protected]>--- makedumpfile.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-)makedumpfile.c +22
-6@@ -32,6 +32,17 @@ int message_level;/* * Forward declarations */ +void print_progress (const char *msg, +unsigned long current, +unsigned long end); + +/* +* Message texts +*/ +#define PROGRESS_COPY "Copying data" +#define PROGRESS_UNN_PAGES "Excluding unnecessary pages" +#define PROGRESS_ZERO_PAGES "Excluding zero pages" +#define PROGRESS_MAXLEN "35"/* * The numbers of the excluded pages @@ -4062,6 +4073,9 @@ exclude_zero_pages() }for (pfn = paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_ZERO_PAGES , pfn, info->max_mapnr); +if (!is_in_segs (paddr))continue ;@@ -4100,6 +4114,8 @@ exclude_unnecessary_pages() goto out; }for (mm =0 ; mm < info->num_mem_map; mm++) { +print_progress (PROGRESS_UNN_PAGES , mm, info->num_mem_map); + mmd = &info->mem_map_data[mm]; pfn = mmd->pfn_start; paddr = pfn*info->page_size;@@ -4670,7 +4686,7 @@ write_kdump_header() }void -print_progress (unsigned long current,unsigned long end) +print_progress (const char *msg,unsigned long current,unsigned long end) {int progress;time_t tm;@@ -4686,7 +4702,7 @@ print_progress(unsigned long current, un progress =100 ;PROGRESS_MSG ("\r" ); -PROGRESS_MSG ("[%3d %%]" , progress); +PROGRESS_MSG ("%-" PROGRESS_MAXLEN "s: [%3d %%] " , msg, progress); }int @@ -4912,7 +4928,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;@@ -5009,7 +5025,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;@@ -5038,7 +5054,7 @@ write_elf_pages()if (!write_cache_bufsz (&cd_seg)) goto out; -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;@@ -5247,7 +5263,7 @@ write_kdump_pages()for (pfn =0 ; pfn < info->max_mapnr; pfn++) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if ((pfn %PFN_BUFBITMAP ) ==0 ) {if (info->len_bitmap - bm2.offset <BUFSIZE_BITMAP )
makedumpfile.c write_kdump_pages() · line 5347
Quoted 8 lines
makedumpfile.c +1
-1@@ -5325,7 +5341,7 @@ write_kdump_pages()/* * Print the progress of the end. */ -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Hi Bernhard,
Quoted 6 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.
Thanks
Ken'ichi Ohmichi
---
makedumpfile.c +44 -7
@@ -32,6 +32,18 @@ int message_level;/* * Forward declarations */ +void print_progress (const char *msg, +unsigned long current, +unsigned long end); + +/* +* Message texts +*/ +#define PROGRESS_COPY "Copying data" +#define PROGRESS_UNN_PAGES "Excluding unnecessary pages" +#define PROGRESS_FREE_PAGES "Excluding free pages" +#define PROGRESS_ZERO_PAGES "Excluding zero pages" +#define PROGRESS_MAXLEN "35"/* * The numbers of the excluded pages
@@ -3917,6 +3929,8 @@ _exclude_free_page() } for (num_nodes = 1; num_nodes <= vt.numnodes; num_nodes++) { + print_progress(PROGRESS_FREE_PAGES, num_nodes - 1, vt.numnodes); + node_zones = pgdat + OFFSET(pglist_data.node_zones); if (!readmem(VADDR, pgdat + OFFSET(pglist_data.nr_zones),
@@ -3948,6 +3962,12 @@ _exclude_free_page() } } } + +/* +* print [100 %] +*/ +print_progress (PROGRESS_FREE_PAGES , vt.numnodes, vt.numnodes); +return TRUE; }
@@ -4062,6 +4082,9 @@ exclude_zero_pages() }for (pfn = paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_ZERO_PAGES , pfn, info->max_mapnr); +if (!is_in_segs (paddr))continue ;
@@ -4073,6 +4096,12 @@ exclude_zero_pages() pfn_zero++; } } + +/* +* print [100 %] +*/ +print_progress (PROGRESS_ZERO_PAGES , info->max_mapnr, info->max_mapnr); + ret = TRUE; out:if (buf !=NULL )
@@ -4100,6 +4129,8 @@ exclude_unnecessary_pages() goto out; }for (mm =0 ; mm < info->num_mem_map; mm++) { +print_progress (PROGRESS_UNN_PAGES , mm, info->num_mem_map); + mmd = &info->mem_map_data[mm]; pfn = mmd->pfn_start; paddr = pfn*info->page_size;
@@ -4160,6 +4191,12 @@ exclude_unnecessary_pages() } } } + +/* +* print [100 %] +*/ +print_progress (PROGRESS_UNN_PAGES , info->num_mem_map, info->num_mem_map); +if (info->dump_level &DL_EXCLUDE_FREE )if (!exclude_free_page ()) goto out;
@@ -4670,7 +4707,7 @@ write_kdump_header() }void -print_progress (unsigned long current,unsigned long end) +print_progress (const char *msg,unsigned long current,unsigned long end) {int progress;time_t tm;
@@ -4686,7 +4723,7 @@ print_progress(unsigned long current, un progress =100 ;PROGRESS_MSG ("\r" ); -PROGRESS_MSG ("[%3d %%]" , progress); +PROGRESS_MSG ("%-" PROGRESS_MAXLEN "s: [%3d %%] " , msg, progress); }int
@@ -4912,7 +4949,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;
@@ -5009,7 +5046,7 @@ write_elf_pages()while (bufsz_remain >0 ) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if (bufsz_remain >= page_size) bufsz_write = page_size;
@@ -5038,7 +5075,7 @@ write_elf_pages()if (!write_cache_bufsz (&cd_seg)) goto out; -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;
@@ -5247,7 +5284,7 @@ write_kdump_pages()for (pfn =0 ; pfn < info->max_mapnr; pfn++) {if ((num_dumped % per) ==0 ) -print_progress (num_dumped, num_dumpable); +print_progress (PROGRESS_COPY , num_dumped, num_dumpable);if ((pfn %PFN_BUFBITMAP ) ==0 ) {if (info->len_bitmap - bm2.offset <BUFSIZE_BITMAP )
@@ -5325,7 +5362,7 @@ write_kdump_pages()/* * Print the progress of the end. */ -print_progress (num_dumpable, num_dumpable); +print_progress (PROGRESS_COPY , num_dumpable, num_dumpable);PROGRESS_MSG ("\n" ); ret = TRUE;
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Hi,
Quoted 6 lines · Ken'ichi Ohmichi
Quoted 5 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)
Quoted 8 lines
Quoted 1 line
I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.
That patch does not apply any more to the latest release. Maybe you can send me your whole tarball?
Bernhard
Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Development _______________________________________________ kexec mailing list [email protected] http://lists.infradead.org/mailman/listinfo/kexec
Hi Bernhard,
Quoted 10 lines · Bernhard Walle
Quoted 9 lines · Ken'ichi Ohmichi
Quoted 6 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
Quoted 7 lines
Quoted 4 lines
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.That patch does not apply any more to the latest release. Maybe you can send me your whole tarball?
Sorry for wasting your time.
OK, I'll send you tarball later.
In the other way, you can get it by the following command:
cvs -z3 -d:pserver:[email protected]:/cvsroot/makedumpfile co -r DEVEL -P makedumpfile
Thanks
Ken'ichi Ohmichi
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Quoted 16 lines · Ken'ichi Ohmichi
Quoted 15 lines · Bernhard Walle
Quoted 13 lines · Ken'ichi Ohmichi
Quoted 6 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.
Quoted 7 lines
Quoted 2 lines
That patch does not apply any more to the latest release. Maybe you can send me your whole tarball?
Sorry for wasting your time.
OK, I'll send you tarball later.
In the other way, you can get it by the following command:
cvs -z3 -d:pserver:[email protected]:/cvsroot/makedumpfile co -r DEVEL -P makedumpfile
Ah, cool, didn't know that a public CVS exists. :) Yes, that works without problems here and it looks nice.
Bernhard
Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Development _______________________________________________ kexec mailing list [email protected] http://lists.infradead.org/mailman/listinfo/kexec
Quoted 6 lines · Ken'ichi Ohmichi
Quoted 5 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)
Quoted 8 lines
Quoted 1 line
I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.
I also tested on a larger SGI machine. Here the exclusion of the memory holes takes a considerable amount of time. Therefore, I would suggest to also use the patch below:
Index: makedumpfile.c
===================================================================
RCS file: /cvsroot/makedumpfile/makedumpfile/makedumpfile.c,v retrieving revision 1.7.2.36
makedumpfile.c +10 -0
@@ -40,6 +40,7 @@* Message texts */ #define PROGRESS_COPY "Copying data" +#define PROGRESS_HOLES "Checking for memory holes"#define PROGRESS_UNN_PAGES "Excluding unnecessary pages"#define PROGRESS_FREE_PAGES "Excluding free pages"#define PROGRESS_ZERO_PAGES "Excluding zero pages"
@@ -4068,11 +4069,20 @@*/ for (pfn =0 , paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_HOLES , pfn, info->max_mapnr); +if (is_in_segs (paddr))set_bit_on_1st_bitmap (pfn);else pfn_memhole++; } + +/* +* print 100 % +*/ +print_progress (PROGRESS_HOLES , info->max_mapnr, info->max_mapnr); +if (!sync_1st_bitmap ()) goto out;
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Quoted 10 lines · Bernhard Walle
Quoted 9 lines · Ken'ichi Ohmichi
Quoted 6 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
Quoted 8 lines
Quoted 4 lines
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.I also tested on a larger SGI machine. Here the exclusion of the memory holes takes a considerable amount of time. Therefore, I would suggest to also use the patch below:
By "memory holes", did you mean zero value memory or memory gap as you can see from /proc/iomem? For example,
26003000000-26033dfffff : System RAM 26033e00000-260f7ffffff : System RAM <-- big gap b/t this and next 26800000000-268f7ffffff : System RAM
If latter, the gap represents address gap between nodes and are non-existent memory. We do not need to spend time on non-existent memory.
Regards,
- jay
Quoted 41 lines
Signed-off-by Bernhard Walle <[email protected]>Index:
makedumpfile.c
===================================================================
RCS file: /cvsroot/makedumpfile/makedumpfile/makedumpfile.c,v retrieving revision 1.7.2.36
diff -u -r1.7.2.36makedumpfile.cmakedumpfile.c +10
-0@@ -40,6 +40,7 @@* Message texts */ #define PROGRESS_COPY "Copying data" +#define PROGRESS_HOLES "Checking for memory holes"#define PROGRESS_UNN_PAGES "Excluding unnecessary pages"#define PROGRESS_FREE_PAGES "Excluding free pages"#define PROGRESS_ZERO_PAGES "Excluding zero pages"@@ -4068,11 +4069,20 @@*/ for (pfn =0 , paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_HOLES , pfn, info->max_mapnr); +if (is_in_segs (paddr))set_bit_on_1st_bitmap (pfn);else pfn_memhole++; } + +/* +* print 100 % +*/ +print_progress (PROGRESS_HOLES , info->max_mapnr, info->max_mapnr); +if (!sync_1st_bitmap ()) goto out;
Quoted 4 lines
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
Hi,
Quoted 9 lines · Jay Lan
By "memory holes", did you mean zero value memory or memory gap as you can see from
/proc/iomem? For example,
26003000000-26033dfffff : System RAM 26033e00000-260f7ffffff : System RAM <-- big gap b/t this and next 26800000000-268f7ffffff : System RAMIf latter, the gap represents address gap between nodes and are non-existent memory.
Right.
Quoted 1 line
We do not need to spend time on non-existent memory.
See the loop below. The code processes very PFN and checks for it
3488 static inline int
3489 is_in_segs(unsigned long long paddr)
3490 {
3491 if (paddr_to_offset(paddr))
3492 return TRUE;
3493 else
3494 return FALSE;
3495 }The whole process takes about 1 minute on a machine with 14 GiB memory. So it's worth printing a progress bar. (Proabably that code could be optimised, but that's an additional step.)
Bernhard
Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Development _______________________________________________ kexec mailing list [email protected] http://lists.infradead.org/mailman/listinfo/kexec
Hi Bernhard,
Quoted 10 lines · Bernhard Walle
Quoted 9 lines · Ken'ichi Ohmichi
Quoted 6 lines · Ken'ichi Ohmichi
Hi Bernhard,
Thank you for your patch.
I like this idea :-)I am busy now, and I will consider the patch well the next week.
Thank you for the patch, and sorry for my late response.
Quoted 8 lines
Quoted 4 lines
I added the progress bar for excluding free pages to your patch. Could you please check the attached patch ?
If there is no problem in the attached patch, I will release the next release with this patch.I also tested on a larger SGI machine. Here the exclusion of the memory holes takes a considerable amount of time. Therefore, I would suggest to also use the patch below:
Thank you for the patch.
It sounds good to me.
I'll merge it into the next release.
Thanks
Ken'ichi Ohmichi
Quoted 29 lines
Signed-off-by Bernhard Walle <[email protected]>Index:
makedumpfile.c
===================================================================
RCS file: /cvsroot/makedumpfile/makedumpfile/makedumpfile.c,v retrieving revision 1.7.2.36
diff -u -r1.7.2.36makedumpfile.cmakedumpfile.c +4
-0@@ -40,6 +40,7 @@* Message texts */ #define PROGRESS_COPY "Copying data" +#define PROGRESS_HOLES "Checking for memory holes"#define PROGRESS_UNN_PAGES "Excluding unnecessary pages"#define PROGRESS_FREE_PAGES "Excluding free pages"#define PROGRESS_ZERO_PAGES "Excluding zero pages"@@ -4068,11 +4069,20 @@*/ for (pfn =0 , paddr =0 ; pfn < info->max_mapnr; pfn++, paddr += info->page_size) { + +print_progress (PROGRESS_HOLES , pfn, info->max_mapnr); +if (is_in_segs (paddr))set_bit_on_1st_bitmap (pfn);else pfn_memhole++;
makedumpfile.c line 4088
Quoted 10 lines
makedumpfile.c +6
-0} + +/* +* print 100 % +*/ +print_progress (PROGRESS_HOLES , info->max_mapnr, info->max_mapnr); +if (!sync_1st_bitmap ()) goto out;
_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec