xen: debugtrace cleanup and per-cpu buffer support
cover xen: debugtrace cleanup and per-cpu buffer support
Another debugtrace enhancement I needed for core scheduling debugging.
Juergen Gross (3):
xen: move debugtrace coding to common/debugtrace.c
xen: refactor debugtrace data
xen: add per-cpu buffer option to debugtrace
docs/misc/xen-command-line.pandoc | 7 +- xen/common/Makefile | 1 + xen/common/debugtrace.c | 262 ++++++++++++++++++++++++++++++++++++++ xen/drivers/char/console.c | 176 +------------------------ xen/include/xen/console.h | 4 + 5 files changed, 273 insertions(+), 177 deletions(-) create mode 100644 xen/common/debugtrace.c
Signature · 7 lines
2.16.4 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 1/3 xen: move debugtrace coding to common/debugtrace.c
Instead of living in drivers/char/console.c move the debugtrace related coding to a new file common/debugtrace.c
--- xen/common/Makefile | 1 + xen/common/debugtrace.c | 177 +++++++++++++++++++++++++++++++++++++++++++++ xen/drivers/char/console.c | 176 +------------------------------------------- xen/include/xen/console.h | 4 + 4 files changed, 184 insertions(+), 174 deletions(-) create mode 100644 xen/common/debugtrace.c
xen/common/Makefile +1 -0
@@ -4,6 +4,7 @@ obj-y += bsearch.o obj-$(CONFIG_CORE_PARKING) += core_parking.o obj-y += cpu.o obj-y += cpupool.o +obj-$(CONFIG_DEBUG_TRACE) += debugtrace.o obj-$(CONFIG_HAS_DEVICE_TREE) += device_tree.o obj-y += domctl.o obj-y += domain.o
xen/common/debugtrace.c new file +177 -0
177 lines not on this page · raw message
xen/drivers/char/console.c +2 -174
194 lines not on this page · raw message
xen/include/xen/console.h +4 -0
@@ -48,4 +48,8 @@ int console_resume(void);extern int8_t opt_console_xen; +/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s); +#endif /* __CONSOLE_H__ */
Signature · 7 lines
2.16.4 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 1/3] xen: move debugtrace coding to common/debugtrace.c
xen/drivers/char/console.c int printk_ratelimit(void) · line 1162
Quoted 9 lines · Juergen Gross
xen/drivers/char/console.c +0
-5@@ -1155,178 +1155,6 @@ int printk_ratelimit(void)return __printk_ratelimit (printk_ratelimit_ms, printk_ratelimit_burst); } -/* -* ************************************************************** -* *************** Serial console ring buffer ******************* -* ************************************************************** -*/
I acknowledge that this comment has at best been displaced from what it would properly belong to, so is indeed perhaps best dropped. But ...
Quoted 159 lines
xen/drivers/char/console.c +0
-159-#ifdef CONFIG_DEBUG_TRACE - -/* Send output direct to console, or buffer it? */ -static volatile int debugtrace_send_to_console; - -static char *debugtrace_buf;/* Debug-trace buffer */ -static unsigned int debugtrace_prd;/* Producer index */ -static unsigned int debugtrace_kilobytes =128 , debugtrace_bytes; -static unsigned int debugtrace_used; -static DEFINE_SPINLOCK (debugtrace_lock); -integer_param ("debugtrace" , debugtrace_kilobytes); - -static void debugtrace_dump_worker (void ) -{ -if ( (debugtrace_bytes ==0 ) || !debugtrace_used ) -return ; - -printk ("debugtrace_dump() starting\n" ); - -/* Print oldest portion of the ring. */ -ASSERT (debugtrace_buf[debugtrace_bytes -1 ] ==0 ); -sercon_puts (&debugtrace_buf[debugtrace_prd]); - -/* Print youngest portion of the ring. */ - debugtrace_buf[debugtrace_prd] ='\0' ; -sercon_puts (&debugtrace_buf[0 ]); - -memset (debugtrace_buf,'\0' , debugtrace_bytes); - -printk ("debugtrace_dump() finished\n" ); -} - -static void debugtrace_toggle (void ) -{ -unsigned long flags; - -watchdog_disable (); -spin_lock_irqsave (&debugtrace_lock, flags); - -/* -* Dump the buffer *before* toggling, in case the act of dumping the -* buffer itself causes more printk() invocations. -*/ -printk ("debugtrace_printk now writing to %s.\n" , - !debugtrace_send_to_console ?"console" :"buffer" ); -if ( !debugtrace_send_to_console ) -debugtrace_dump_worker (); - - debugtrace_send_to_console = !debugtrace_send_to_console; - -spin_unlock_irqrestore (&debugtrace_lock, flags); -watchdog_enable (); - -} - -void debugtrace_dump (void ) -{ -unsigned long flags; - -watchdog_disable (); -spin_lock_irqsave (&debugtrace_lock, flags); - -debugtrace_dump_worker (); - -spin_unlock_irqrestore (&debugtrace_lock, flags); -watchdog_enable (); -} - -static void debugtrace_add_to_buf (char *buf) -{ -char *p; - -for ( p = buf; *p !='\0' ; p++ ) - { - debugtrace_buf[debugtrace_prd++] = *p; -/* Always leave a nul byte at the end of the buffer. */ -if ( debugtrace_prd == (debugtrace_bytes -1 ) ) - debugtrace_prd =0 ; - } -} - -void debugtrace_printk (const char *fmt, ...) -{ -static char buf[1024 ], last_buf[1024 ]; -static unsigned int count, last_count, last_prd; - -char cntbuf[24 ]; -va_list args; -unsigned long flags; - -if ( debugtrace_bytes ==0 ) -return ; - - debugtrace_used =1 ; - -spin_lock_irqsave (&debugtrace_lock, flags); - -ASSERT (debugtrace_buf[debugtrace_bytes -1 ] ==0 ); - -va_start (args, fmt); -vsnprintf (buf,sizeof (buf), fmt, args); -va_end (args); - -if ( debugtrace_send_to_console ) - { -snprintf (cntbuf,sizeof (cntbuf),"%u " , ++count); -serial_puts (sercon_handle, cntbuf); -serial_puts (sercon_handle, buf); - } -else - { -if (strcmp (buf, last_buf) ) - { - last_prd = debugtrace_prd; - last_count = ++count; -safe_strcpy (last_buf, buf); -snprintf (cntbuf,sizeof (cntbuf),"%u " , count); - } -else - { - debugtrace_prd = last_prd; -snprintf (cntbuf,sizeof (cntbuf),"%u-%u " , last_count, ++count); - } -debugtrace_add_to_buf (cntbuf); -debugtrace_add_to_buf (buf); - } - -spin_unlock_irqrestore (&debugtrace_lock, flags); -} - -static void debugtrace_key (unsigned char key) -{ -debugtrace_toggle (); -} - -static int __init debugtrace_init (void ) -{ -int order; -unsigned int kbytes, bytes; - -/* Round size down to next power of two. */ -while ( (kbytes = (debugtrace_kilobytes & (debugtrace_kilobytes-1 ))) !=0 ) - debugtrace_kilobytes = kbytes; - - bytes = debugtrace_kilobytes <<10 ; -if ( bytes ==0 ) -return 0 ; - - order =get_order_from_bytes (bytes); - debugtrace_buf =alloc_xenheap_pages (order,0 ); -ASSERT (debugtrace_buf !=NULL ); - -memset (debugtrace_buf,'\0' , bytes); - - debugtrace_bytes = bytes; - -register_keyhandler ('T' , debugtrace_key, -"toggle debugtrace to console/buffer" ,0 ); -
xen/drivers/char/console.c int printk_ratelimit(void) · line 1160
Quoted 10 lines
xen/drivers/char/console.c +0
-7- return 0; -} -__initcall (debugtrace_init); - -#endif /* !CONFIG_DEBUG_TRACE */ - -/* * ************************************************************** * *************** Debugging/tracing/error-report ***************
... what about this one? There's only panic() between it and the next such comment, and I don't think the "Debugging/tracing" part of it are applicable (anymore).
xen/include/xen/console.h int console_resume(void); · line 53
Quoted 9 lines
xen/include/xen/console.h +3
-0@@ -48,4 +48,8 @@ int console_resume(void);extern int8_t opt_console_xen; +/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s);
I guess avoiding their exposure was one of the reasons the debug trace code lived in the place you move it from. I'm unconvinced non-console code is actually supposed to make use of either, but I'm not opposed enough to nak the change. I don't think though the comment fits well with the variable declaration.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 1/3] xen: move debugtrace coding to common/debugtrace.c
Quoted 11 lines · Jan Beulich
xen/drivers/char/console.c int printk_ratelimit(void) · line 1160
Quoted 6 lines · Juergen Gross
xen/drivers/char/console.c +0
-3-#endif /* !CONFIG_DEBUG_TRACE */ - -/* * ************************************************************** * *************** Debugging/tracing/error-report *************** ... what about this one? There's only
panic()between it and the next such comment, and I don't think the "Debugging/tracing" part of it are applicable (anymore).
True. I'll remove the "Debugging/tracing" part.
Quoted 7 lines
Quoted 7 lines
xen/include/xen/console.h +0
-0@@ -48,4 +48,8 @@ int console_resume(void);extern int8_t opt_console_xen;
Quoted 9 lines
xen/include/xen/console.h int console_resume(void); · line 53
Quoted 3 lines
xen/include/xen/console.h +3
-0+/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s);I guess avoiding their exposure was one of the reasons the debug trace code lived in the place you move it from. I'm unconvinced non-console code is actually supposed to make use of either, but I'm not opposed enough to nak the change. I don't think though the comment fits well with the variable declaration.
sercon_handle is used for calling serial_puts(), so maybe instead of directly using serial_puts() with sercon_handle I should add a wrapper to console.c (e.g. console_serial_puts())? It should be noted that serial_puts() is called only in case of debugtrace output toggled to go to the console. I guess using serial_puts() in that case is meant to avoid too many software layers when doing the output. It would be possible to use sercon_puts() for that case, too, resulting in the inability to use debugtrace_printk() in the then additionally needed paths (or better: to use it with output redirected to console).
sercon_puts() could use another wrapper, e.g. console_debug_puts().
Would you like that better?
Juergen
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 1/3] xen: move debugtrace coding to common/debugtrace.c
Quoted 30 lines · Juergen Gross
Quoted 11 lines · Jan Beulich
Quoted 6 lines · Juergen Gross
xen/drivers/char/console.c +0
-3-#endif /* !CONFIG_DEBUG_TRACE */ - -/*
* **************************************************************
* *************** Debugging/tracing/error-report ***************... what about this one? There's only
panic()between it and the next such comment, and I don't think the "Debugging/tracing" part of it are applicable (anymore).True. I'll remove the "Debugging/tracing" part.
Quoted 14 lines
xen/include/xen/console.h int console_resume(void); · line 51
Quoted 8 lines
xen/include/xen/console.h +3
-0@@ -48,4 +48,8 @@ int console_resume(void); extern int8_t opt_console_xen; +/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s);I guess avoiding their exposure was one of the reasons the debug trace code lived in the place you move it from. I'm unconvinced non-console code is actually supposed to make use of either, but I'm not opposed enough to nak the change. I don't think though the comment fits well with the variable declaration.
Quoted 6 lines
sercon_handleis used for callingserial_puts(), so maybe instead of directly usingserial_puts()withsercon_handleI should add a wrapper toconsole.c(e.g.console_serial_puts())? It should be noted thatserial_puts()is called only in case of debugtrace output toggled to go to the console. I guess usingserial_puts()in that case is meant to avoid too many software layers when doing the output.
Hmm, I'd rather expect this to be used to avoid doing anything else sercon_puts() does besides calling serial_puts(). These other steps are also why I think this is to remain a console internal interface.
Quoted 8 lines
It would be
possible to usesercon_puts()for that case, too, resulting in the inability to usedebugtrace_printk()in the then additionally needed paths (or better: to use it with output redirected to console).
sercon_puts()could use another wrapper, e.g.console_debug_puts().Would you like that better?
Probably not. I wonder whether splitting out this code is really a good step.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 1/3] xen: move debugtrace coding to common/debugtrace.c
Quoted 38 lines · Jan Beulich
Quoted 30 lines · Juergen Gross
Quoted 11 lines · Jan Beulich
xen/drivers/char/console.c int printk_ratelimit(void) · line 1160
Quoted 6 lines · Juergen Gross
xen/drivers/char/console.c +0
-3-#endif /* !CONFIG_DEBUG_TRACE */ - - /** ************************************************************** * *************** Debugging/tracing/error-report *************** ... what about this one? There's only
panic()between it and the next such comment, and I don't think the "Debugging/tracing" part of it are applicable (anymore).True. I'll remove the "Debugging/tracing" part.
Quoted 14 lines
xen/include/xen/console.h int console_resume(void); · line 51
Quoted 8 lines
xen/include/xen/console.h +3
-0@@ -48,4 +48,8 @@ int console_resume(void); extern int8_t opt_console_xen; +/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s);I guess avoiding their exposure was one of the reasons the debug trace code lived in the place you move it from. I'm unconvinced non-console code is actually supposed to make use of either, but I'm not opposed enough to nak the change. I don't think though the comment fits well with the variable declaration.
Quoted 6 lines
sercon_handleis used for callingserial_puts(), so maybe instead of directly usingserial_puts()withsercon_handleI should add a wrapper toconsole.c(e.g.console_serial_puts())? It should be noted thatserial_puts()is called only in case of debugtrace output toggled to go to the console. I guess usingserial_puts()in that case is meant to avoid too many software layers when doing the output.
Quoted 3 lines
Hmm, I'd rather expect this to be used to avoid doing anything else
sercon_puts()does besides callingserial_puts(). These other steps are also why I think this is to remain a console internal interface.
To me it seems a little bit strange to have the buffer dumping using sercon_puts() while issuing the actual trace entries to console isn't using it.
Quoted 6 lines
Quoted 6 lines
It would be
possible to usesercon_puts()for that case, too, resulting in the inability to usedebugtrace_printk()in the then additionally needed paths (or better: to use it with output redirected to console).
Quoted 6 lines
Quoted 3 lines
sercon_puts()could use another wrapper, e.g.console_debug_puts().Would you like that better?
Probably not. I wonder whether splitting out this code is really a good step.
I'm not fighting for it, I just thought it would better be put into a file of its own.
In case you disagree and others are not pushing for separation I'm fine to drop this patch.
Juergen
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 1/3] xen: move debugtrace coding to common/debugtrace.c
Quoted 39 lines · Juergen Gross
Quoted 38 lines · Jan Beulich
Quoted 36 lines · Juergen Gross
Quoted 11 lines · Jan Beulich
Quoted 6 lines · Juergen Gross
xen/drivers/char/console.c +0
-3-#endif /* !CONFIG_DEBUG_TRACE */ - -/*
* **************************************************************
* *************** Debugging/tracing/error-report ***************... what about this one? There's only
panic()between it and the next such comment, and I don't think the "Debugging/tracing" part of it are applicable (anymore).True. I'll remove the "Debugging/tracing" part.
Quoted 14 lines
xen/include/xen/console.h int console_resume(void); · line 51
Quoted 8 lines
xen/include/xen/console.h +3
-0@@ -48,4 +48,8 @@ int console_resume(void); extern int8_t opt_console_xen; +/* Issue string via serial line. */ +extern int sercon_handle; +void sercon_puts (const char *s);I guess avoiding their exposure was one of the reasons the debug trace code lived in the place you move it from. I'm unconvinced non-console code is actually supposed to make use of either, but I'm not opposed enough to nak the change. I don't think though the comment fits well with the variable declaration.
sercon_handleis used for callingserial_puts(), so maybe instead of directly usingserial_puts()withsercon_handleI should add a wrapper toconsole.c(e.g.console_serial_puts())? It should be noted thatserial_puts()is called only in case of debugtrace output toggled to go to the console. I guess usingserial_puts()in that case is meant to avoid too many software layers when doing the output.
Quoted 7 lines
Quoted 3 lines
Hmm, I'd rather expect this to be used to avoid doing anything else
sercon_puts()does besides callingserial_puts(). These other steps are also why I think this is to remain a console internal interface.To me it seems a little bit strange to have the buffer dumping using
sercon_puts()while issuing the actual trace entries to console isn't using it.
I guess I agree.
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines
It would be
possible to usesercon_puts()for that case, too, resulting in the inability to usedebugtrace_printk()in the then additionally needed paths (or better: to use it with output redirected to console).
sercon_puts()could use another wrapper, e.g.console_debug_puts().Would you like that better?
Quoted 8 lines
Quoted 2 lines
Probably not. I wonder whether splitting out this code is really a good step.
I'm not fighting for it, I just thought it would better be put into a file of its own.
In case you disagree and others are not pushing for separation I'm fine to drop this patch.
Well, I don't mind the separation as long as it's indeed properly separated in the end.
Jan
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 2/3 xen: refactor debugtrace data
In reply to Juergen GrossAs a preparation for per-cpu buffers do a little refactoring of the debugtrace data: put the needed buffer admin data into the buffer as it will be needed for each buffer.
While at it switch debugtrace_send_to_console and debugtrace_used to bool and delete an empty line.
--- xen/common/debugtrace.c | 58 ++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 25 deletions(-)
xen/common/debugtrace.c +33 -25
@@ -15,31 +15,37 @@#include <xen/watchdog.h> /* Send output direct to console, or buffer it? */ -static volatile int debugtrace_send_to_console; +static volatile bool debugtrace_send_to_console; -static char *debugtrace_buf;/* Debug-trace buffer */ -static unsigned int debugtrace_prd;/* Producer index */ -static unsigned int debugtrace_kilobytes =128 , debugtrace_bytes; -static unsigned int debugtrace_used; +struct debugtrace_data_s { +unsigned long bytes;/* Size of buffer. */ +unsigned long prd;/* Producer index. */ +char buf[]; +}; + +static struct debugtrace_data_s *debtr_data; + +static unsigned int debugtrace_kilobytes =128 ; +static bool debugtrace_used;static DEFINE_SPINLOCK (debugtrace_lock);integer_param ("debugtrace" , debugtrace_kilobytes);static void debugtrace_dump_worker (void ) { -if ( (debugtrace_bytes ==0 ) || !debugtrace_used ) +if ( !debtr_data || !debugtrace_used )return ;printk ("debugtrace_dump() starting\n" );/* Print oldest portion of the ring. */ -ASSERT (debugtrace_buf[debugtrace_bytes -1 ] ==0 ); -sercon_puts (&debugtrace_buf[debugtrace_prd]); +ASSERT (debtr_data->buf[debtr_data->bytes -1 ] ==0 ); +sercon_puts (&debtr_data->buf[debtr_data->prd]);/* Print youngest portion of the ring. */ - debugtrace_buf[debugtrace_prd] ='\0' ; -sercon_puts (&debugtrace_buf[0 ]); + debtr_data->buf[debtr_data->prd] ='\0' ; +sercon_puts (&debtr_data->buf[0 ]); -memset (debugtrace_buf,'\0' , debugtrace_bytes); +memset (debtr_data->buf,'\0' , debtr_data->bytes);printk ("debugtrace_dump() finished\n" ); }
@@ -64,7 +70,6 @@ static void debugtrace_toggle(void)spin_unlock_irqrestore (&debugtrace_lock, flags);watchdog_enable (); - }void debugtrace_dump (void )
@@ -86,10 +91,10 @@ static void debugtrace_add_to_buf(char *buf)for ( p = buf; *p !='\0' ; p++ ) { - debugtrace_buf[debugtrace_prd++] = *p; + debtr_data->buf[debtr_data->prd++] = *p;/* Always leave a nul byte at the end of the buffer. */ -if ( debugtrace_prd == (debugtrace_bytes -1 ) ) - debugtrace_prd =0 ; +if ( debtr_data->prd == (debtr_data->bytes -1 ) ) + debtr_data->prd =0 ; } }
@@ -102,14 +107,14 @@ void debugtrace_printk(const char *fmt, ...)va_list args;unsigned long flags; -if ( debugtrace_bytes ==0 ) +if ( !debtr_data )return ; - debugtrace_used =1 ; + debugtrace_used = true;spin_lock_irqsave (&debugtrace_lock, flags); -ASSERT (debugtrace_buf[debugtrace_bytes -1 ] ==0 ); +ASSERT (debtr_data->buf[debtr_data->bytes -1 ] ==0 );va_start (args, fmt);vsnprintf (buf,sizeof (buf), fmt, args);
@@ -125,14 +130,14 @@ void debugtrace_printk(const char *fmt, ...) {if (strcmp (buf, last_buf) ) { - last_prd = debugtrace_prd; + last_prd = debtr_data->prd; last_count = ++count;safe_strcpy (last_buf, buf);snprintf (cntbuf,sizeof (cntbuf),"%u " , count); }else { - debugtrace_prd = last_prd; + debtr_data->prd = last_prd;snprintf (cntbuf,sizeof (cntbuf),"%u-%u " , last_count, ++count); }debugtrace_add_to_buf (cntbuf);
@@ -150,7 +155,8 @@ static void debugtrace_key(unsigned char key)static int __init debugtrace_init (void ) {int order; -unsigned int kbytes, bytes; +unsigned long kbytes, bytes; +struct debugtrace_data_s *data;/* Round size down to next power of two. */ while ( (kbytes = (debugtrace_kilobytes & (debugtrace_kilobytes-1 ))) !=0 )
@@ -161,12 +167,14 @@ static int __init debugtrace_init(void)return 0 ; order =get_order_from_bytes (bytes); - debugtrace_buf =alloc_xenheap_pages (order,0 ); -ASSERT (debugtrace_buf !=NULL ); + data =alloc_xenheap_pages (order,0 ); +if ( !data ) +return -ENOMEM ; -memset (debugtrace_buf,'\0' , bytes); +memset (data,'\0' , bytes); - debugtrace_bytes = bytes; + data->bytes = bytes -sizeof (*data); + debtr_data = data;register_keyhandler ('T' , debugtrace_key,"toggle debugtrace to console/buffer" ,0 );
Signature · 7 lines
2.16.4 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 3/3 xen: add per-cpu buffer option to debugtrace
In reply to Juergen Grossdebugtrace is normally writing trace entries into a single trace buffer. There are cases where this is not optimal, e.g. when hunting a bug which requires writing lots of trace entries and one cpu is stuck. This will result in other cpus filling the trace buffer and finally overwriting the interesting trace entries of the hanging cpu.
In order to be able to debug such situations add the capability to use per-cpu trace buffers. This can be selected by specifying the debugtrace boot parameter with the modifier "cpu:", like:
debugtrace=cpu:16
At the same time switch the parsing function to accept size modifiers (e.g. 4M or 1G).
Printing out the trace entries is done for each buffer in order to minimize the effort needed during printing. As each entry is prefixed with its sequence number sorting the entries can easily be done when analyzing them.
--- docs/misc/xen-command-line.pandoc | 7 +- xen/common/debugtrace.c | 145 +++++++++++++++++++++++++++++--------- 2 files changed, 115 insertions(+), 37 deletions(-)
docs/misc/xen-command-line.pandoc +4 -3
@@ -644,12 +644,13 @@ over the PCI busses sequentially) or by PCI device (must be on segment 0). Limits the number lines printed in Xen stack traces. ### debugtrace -> `= <integer>` +> `= [cpu:]<size>` > Default: `128` -Specify the size of the console debug trace buffer in KiB. The debug -trace feature is only enabled in debugging builds of Xen. +Specify the size of the console debug trace buffer. By specifying `cpu:` +additionally a trace buffer of the specified size is allocated per cpu. +The debug trace feature is only enabled in debugging builds of Xen. ### dma_bits > `= <integer>`
xen/common/debugtrace.c +111 -34
230 lines not on this page · raw message
Signature · 7 lines
2.16.4 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
Re: [Xen-devel] [PATCH 3/3] xen: add per-cpu buffer option to debugtrace
Quoted 238 lines · Juergen Gross
debugtrace is normally writing trace entries into a single trace buffer. There are cases where this is not optimal, e.g. when hunting a bug which requires writing lots of trace entries and one cpu is stuck. This will result in other cpus filling the trace buffer and finally overwriting the interesting trace entries of the hanging cpu.
In order to be able to debug such situations add the capability to use per-cpu trace buffers. This can be selected by specifying the debugtrace boot parameter with the modifier "cpu:", like:
debugtrace=cpu:16
At the same time switch the parsing function to accept size modifiers (e.g. 4M or 1G).
Printing out the trace entries is done for each buffer in order to minimize the effort needed during printing. As each entry is prefixed with its sequence number sorting the entries can easily be done when analyzing them.
Signed-off-by Juergen Gross <[email protected]>--- docs/misc/xen-command-line.pandoc | 7 +- xen/common/debugtrace.c | 145 +++++++++++++++++++++++++++++--------- 2 files changed, 115 insertions(+), 37 deletions(-)docs/misc/xen-command-line.pandoc +4
-3@@ -644,12 +644,13 @@ over the PCI busses sequentially) or by PCI device (must be on segment 0). Limits the number lines printed in Xen stack traces. ### debugtrace -> `= <integer>` +> `= [cpu:]<size>` > Default: `128` -Specify the size of the console debug trace buffer in KiB. The debug -trace feature is only enabled in debugging builds of Xen. +Specify the size of the console debug trace buffer. By specifying `cpu:` +additionally a trace buffer of the specified size is allocated per cpu. +The debug trace feature is only enabled in debugging builds of Xen. ### dma_bits > `= <integer>`xen/common/debugtrace.c +75
-31@@ -6,6 +6,7 @@#include <xen/console.h> +#include <xen/cpu.h> #include <xen/init.h> #include <xen/keyhandler.h> #include <xen/lib.h> @@ -24,30 +25,61 @@ struct debugtrace_data_s { };static struct debugtrace_data_s *debtr_data; +static DEFINE_PER_CPU(struct debugtrace_data_s *, debtr_cpu_data); -static unsigned int debugtrace_kilobytes =128 ; +static unsigned int debugtrace_bytes =128 <<10 ; +static bool debugtrace_per_cpu;static bool debugtrace_used;static DEFINE_SPINLOCK (debugtrace_lock); -integer_param ("debugtrace" , debugtrace_kilobytes); -static void debugtrace_dump_worker (void ) +static int __init debugtrace_parse_param (const char *s) +{ +if ( !strncmp (s,"cpu:" ,4 ) ) + { + debugtrace_per_cpu = true; + s +=4 ; + } + debugtrace_bytes =parse_size_and_unit (s,NULL ); +return 0 ; +} +custom_param ("debugtrace" , debugtrace_parse_param); + +static void debugtrace_dump_buffer (struct debugtrace_data_s *data, +const char *which) { -if ( !debtr_data || !debugtrace_used ) +if ( !data )return ; -printk ("debugtrace_dump() starting\n" ); +printk ("debugtrace_dump() %s buffer starting\n" , which);/* Print oldest portion of the ring. */ -ASSERT (debtr_data->buf[debtr_data->bytes -1 ] ==0 ); -sercon_puts (&debtr_data->buf[debtr_data->prd]); +ASSERT (data->buf[data->bytes -1 ] ==0 ); +sercon_puts (&data->buf[data->prd]);/* Print youngest portion of the ring. */ - debtr_data->buf[debtr_data->prd] ='\0' ; -sercon_puts (&debtr_data->buf[0 ]); + data->buf[data->prd] ='\0' ; +sercon_puts (&data->buf[0 ]); + +memset (data->buf,'\0' , data->bytes); + +printk ("debugtrace_dump() %s buffer finished\n" , which); +} + +static void debugtrace_dump_worker (void ) +{ +unsigned int cpu; +char buf[16 ]; + +if ( !debugtrace_used ) +return ; -memset (debtr_data->buf,'\0' , debtr_data->bytes); +debugtrace_dump_buffer (debtr_data,"global" ); -printk ("debugtrace_dump() finished\n" ); +for ( cpu =0 ; cpu < nr_cpu_ids; cpu++ ) + { +snprintf (buf,sizeof (buf),"cpu %u" , cpu); +debugtrace_dump_buffer (per_cpu (debtr_cpu_data, cpu), buf); + } }static void debugtrace_toggle (void )@@ -87,34 +119,41 @@ void debugtrace_dump(void)static void debugtrace_add_to_buf (char *buf) { +struct debugtrace_data_s *data;char *p; + data = debugtrace_per_cpu ?this_cpu (debtr_cpu_data) : debtr_data; +for ( p = buf; *p !='\0' ; p++ ) { - debtr_data->buf[debtr_data->prd++] = *p; + data->buf[data->prd++] = *p;/* Always leave a nul byte at the end of the buffer. */ -if ( debtr_data->prd == (debtr_data->bytes -1 ) ) - debtr_data->prd =0 ; +if ( data->prd == (data->bytes -1 ) ) + data->prd =0 ; } }void debugtrace_printk (const char *fmt, ...) {static char buf[1024 ], last_buf[1024 ]; -static unsigned int count, last_count, last_prd; +static unsigned int count, last_count, last_prd, last_cpu;char cntbuf[24 ];va_list args;unsigned long flags; +struct debugtrace_data_s *data; +unsigned int cpu; -if ( !debtr_data ) + data = debugtrace_per_cpu ?this_cpu (debtr_cpu_data) : debtr_data; + cpu = debugtrace_per_cpu ?smp_processor_id () :0 ; +if ( !data )return ; debugtrace_used = true;spin_lock_irqsave (&debugtrace_lock, flags); -ASSERT (debtr_data->buf[debtr_data->bytes -1 ] ==0 ); +ASSERT (data->buf[data->bytes -1 ] ==0 );va_start (args, fmt);vsnprintf (buf,sizeof (buf), fmt, args);@@ -128,16 +167,17 @@ void debugtrace_printk(const char *fmt, ...) }else { -if (strcmp (buf, last_buf) ) +if (strcmp (buf, last_buf) || cpu != last_cpu ) { - last_prd = debtr_data->prd; + last_prd = data->prd; last_count = ++count; + last_cpu = cpu;safe_strcpy (last_buf, buf);snprintf (cntbuf,sizeof (cntbuf),"%u " , count); }else { - debtr_data->prd = last_prd; + data->prd = last_prd;snprintf (cntbuf,sizeof (cntbuf),"%u-%u " , last_count, ++count); }debugtrace_add_to_buf (cntbuf);@@ -152,33 +192,70 @@ static void debugtrace_key(unsigned char key)debugtrace_toggle (); } -static int __init debugtrace_init (void ) +static void debugtrace_alloc_buffer (struct debugtrace_data_s **ptr) {int order; -unsigned long kbytes, bytes;struct debugtrace_data_s *data; -/* Round size down to next power of two. */ -while ( (kbytes = (debugtrace_kilobytes & (debugtrace_kilobytes-1 ))) !=0 ) - debugtrace_kilobytes = kbytes; - - bytes = debugtrace_kilobytes <<10 ; -if ( bytes ==0 ) -return 0 ; +if ( debugtrace_bytes <PAGE_SIZE ) +return ; - order =get_order_from_bytes (bytes); + order =get_order_from_bytes (debugtrace_bytes); data =alloc_xenheap_pages (order,0 );if ( !data ) -return -ENOMEM ; + { +printk ("failed to allocate debugtrace buffer\n" ); +return ; + } + +memset (data,'\0' , debugtrace_bytes); + data->bytes = debugtrace_bytes -sizeof (*data); + + *ptr = data; +} +
xen/common/debugtrace.c static void debugtrace_key(unsigned char key) · line 226
Quoted 10 lines
xen/common/debugtrace.c +10
-0+static int debugtrace_cpu_callback (struct notifier_block *nfb, +unsigned long action,void *hcpu) +{ +unsigned int cpu = (unsigned long )hcpu; + +/* Buffers are only ever allocated, never freed. */ +switch ( action ) + { +case CPU_UP_PREPARE : +debugtrace_alloc_buffer (&per_cpu (debtr_cpu_data, cpu));
I just realized this is leaking buffers when offlining and onlining a cpu again.
Juergen
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel