Skip to contentSkip to conversation

Linux Kernel Markers 0.2 for Linux 2.6.17

PATCH In review Mathieu Desnoyers 6 participants 18 messages Sep 19, 2006 – Sep 20, 2006

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Linux Kernel Markers 0.2 for Linux 2.6.17

Good afternoon,

Following some very interesting ideas from Martin that shows that even static function calls and inlined functions can be used in interesting ways with markers to deploy dynamic tracers with easy access to local function variables, I send this slightly improved version of Linux Kernel Markers.

It has the same capabilities as the previous one and additionnaly checks for string format consistency in every kernel configuration. The idea behind this is to be told be the compiler as soon as a marker is broken.

These last emails convince me even more that a markup mechanism must interface with every kind of instrumentation hooking we can think about, both dynamic and static.

Mathieu

--- BEGIN ---

arch/i386/Kconfig +2 -0
@@ -1082,6 +1082,8 @@ config KPROBES
 	  for kernel debugging, non-intrusive instrumentation and testing.
 	  If in doubt, say "N".
 
+source "kernel/Kconfig.marker"
+
 source "ltt/Kconfig"
 
 endmenu
include/asm-i386/marker.h new file +12 -0
@@ -0,0 +1,12 @@
+/*****************************************************************************
+ * marker.h
+ *
+ * Code markup for dynamic and static tracing. i386 support.
+ *
+ * Mathieu Desnoyers <[email protected]>
+ *
+ * September 2006
+ */
+
+#define JPROBE_TARGET \
+	__asm__ ( GENERIC_NOP5 )
include/linux/marker.h new file +93 -0

93 lines not on this page · raw message

kernel/Kconfig.marker new file +75 -0

75 lines not on this page · raw message

--- END ---

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#1 lore raw
Quoted 36 lines · Mathieu Desnoyers

Good afternoon,

Following some very interesting ideas from Martin that shows that even static function calls and inlined functions can be used in interesting ways with markers to deploy dynamic tracers with easy access to local function variables, I send this slightly improved version of Linux Kernel Markers.

It has the same capabilities as the previous one and additionnaly checks for string format consistency in every kernel configuration. The idea behind this is to be told be the compiler as soon as a marker is broken.

These last emails convince me even more that a markup mechanism must interface with every kind of instrumentation hooking we can think about, both dynamic and static.

Mathieu

--- BEGIN ---

arch/i386/Kconfig +2 -0
@@ -1082,6 +1082,8 @@ config KPROBES
 	  for kernel debugging, non-intrusive instrumentation and testing.
 	  If in doubt, say "N".
 
+source "kernel/Kconfig.marker"
+
 source "ltt/Kconfig"
 
 endmenu
include/asm-i386/marker.h new file +2 -0
@@ -0,0 +1,12 @@
+/*****************************************************************************
+ * marker.h

include/asm-i386/marker.h line 12

Quoted 10 lines
include/asm-i386/marker.h new file +10 -0
+ *
+ * Code markup for dynamic and static tracing. i386 support.
+ *
+ * Mathieu Desnoyers <[email protected]>
+ *
+ * September 2006
+ */
+
+#define JPROBE_TARGET \
+	__asm__ ( GENERIC_NOP5 )

Too many spaces; use:

	__asm__ (GENERIC_NOP5)
Quoted 84 lines
include/linux/marker.h new file +81 -0
@@ -0,0 +1,93 @@
+/*****************************************************************************
+ * marker.h
+ *
+ * Code markup for dynamic and static tracing.
+ *
+ * Use either :
+ * MARK
+ * MARK_NOPRINT (will never call printk)
+ * MARK_STATIC (not dynamically instrumentable, will never call printk)
+ *
+ * Example :
+ *
+ * MARK(subsystem_event, "%d %s", someint, somestring);
+ * Where :
+ * - Subsystem is the name of your subsystem.
+ * - event is the name of the event to mark.
+ * - "%d %s" is the formatted string for printk.
+ * - someint is an integer.
+ * - somestring is a char *.
+ * - subsystem_event must be unique thorough the kernel!
+ *
+ * Mathieu Desnoyers <[email protected]>
+ *
+ * September 2006
+ */
+
+#include <asm/marker.h>
+
+#define MARK_SYM(event) \
+	__asm__ ( "__mark_" #event ":" )
+
+#define MARK_INACTIVE(event, format, args...) \
+	__mark_check_format(format, ## args)
+
+#define MARK_PRINT(event, format, args...) \
+	do { \
+		__mark_check_format(format, ## args); \
+		printk(#event ": " format, ## args); \
+	} while(0)
+
+#define MARK_FPROBE(event, format, args...) \
+	do { \
+		__mark_check_format(format, ## args); \
+		fprobe_##event(args); \
+	} while(0)
+
+#define MARK_KPROBE(event, format, args...) \
+	do { \
+		__mark_check_format(format, ## args); \
+		MARK_SYM(event); \
+	} while(0)
+
+#define MARK_JPROBE(event, format, args...) \
+	do { \
+		__mark_check_format(format, ## args); \
+		MARK_SYM(event); \
+		JPROBE_TARGET; \
+	} while(0)
+
+/* Menu configured markers */
+#ifndef CONFIG_MARK
+#define MARK	MARK_INACTIVE
+#elif defined(CONFIG_MARK_PRINT)
+#define MARK	MARK_PRINT
+#elif defined(CONFIG_MARK_FPROBE)
+#define MARK	MARK_FPROBE
+#elif defined(CONFIG_MARK_KPROBE)
+#define MARK	MARK_KPROBE
+#elif defined(CONFIG_MARK_JPROBE)
+#define MARK	MARK_JPROBE
+#endif
+
+#ifndef CONFIG_MARK_NOPRINT
+#define MARK_NOPRINT	MARK_INACTIVE
+#elif defined(CONFIG_MARK_NOPRINT_FPROBE)
+#define MARK_NOPRINT	MARK_FPROBE
+#elif defined(CONFIG_MARK_NOPRINT_KPROBE)
+#define MARK_NOPRINT	MARK_KPROBE
+#elif defined(CONFIG_MARK_NOPRINT_JPROBE)
+#define MARK_NOPRINT	MARK_JPROBE
+#endif

include/linux/marker.h line 91

Quoted 10 lines
include/linux/marker.h new file +10 -0
+
+#ifndef CONFIG_MARK_STATIC
+#define MARK_STATIC	MARK_INACTIVE
+#else
+#define MARK_STATIC	MARK_FPROBE
+#endif
+
+static inline void __mark_check_format(const char *fmt, ...)
+	__attribute__ ((format (printf, 1, 2)));
+void __mark_check_format(const char *fmt, ...) {  }

That last line is confusing (to me). What's it for? Is it just an empty (inline) function definition? If so, why repeat the void + function name?

kernel/Kconfig.marker line 8

Quoted 11 lines
kernel/Kconfig.marker new file +8 -0
@@ -0,0 +1,75 @@
+# Code markers configuration
+
+menu "Marker configuration"
+
+
+config MARK
+	bool "Enable MARK code markers"
+	default y

Please justify using 'y' as the default value.

Quoted 6 lines
kernel/Kconfig.marker new file +6 -0
+	help
+	  Activate markers that can call printk or can be instrumented
+	  dynamically.
+
+choice
+	prompt "MARK code marker behavior"

kernel/Kconfig.marker line 24

Quoted 10 lines
kernel/Kconfig.marker new file +10 -0
+	default MARK_KPROBE
+	depends on MARK
+	help
+	  Configuration of markers that can call printk or can be
+	  instrumented dynamically.
+
+config MARK_KPROBE
+	bool "KPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event".

Indent help text by 2 spaces (above & below here).

Quoted 53 lines
kernel/Kconfig.marker new file +51 -1
+config MARK_JPROBE
+	bool "JPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event"
+	and create a target for a high speed dynamic probe.
+config MARK_FPROBE
+	bool "FPROBE"
+	---help---
+	Change markers for a function call.
+config MARK_PRINT
+	bool "PRINT"
+	---help---
+	Call printk from the marker.
+endchoice
+
+config MARK_NOPRINT
+	bool "Enable MARK_NOPRINT code markers"
+	default y
+	help
+	  Activate markers that cannot call printk.
+
+choice
+	prompt "MARK_NOPRINT code marker behavior"
+	default MARK_NOPRINT_KPROBE
+	depends on MARK_NOPRINT
+	help
+	  Configuration of markers that cannot call printk.
+
+config MARK_NOPRINT_KPROBE
+	bool "KPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event".
+config MARK_NOPRINT_JPROBE
+	bool "JPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event"
+	and create a target for a high speed dynamic probe.
+config MARK_NOPRINT_FPROBE
+	bool "FPROBE"
+	---help---
+	Change markers for a function call.
+endchoice
+
+config MARK_STATIC
+	bool "Enable MARK_STATIC code markers"
+	default n
+	help
+	  Activate markers that cannot be instrumented dynamically. They will
+	  generate function calls to each function-style probe.
+
+endmenu
 
--- END ---
Signature · 1 line
~Randy
#3 lore raw
In reply to Mathieu Desnoyers

On Tue, Sep 19, 2006 at 02:34:47PM -0400, Mathieu Desnoyers wrote:
[....]

Quoted 68 lines
kernel/Kconfig.marker new file +65 -0
@@ -0,0 +1,75 @@
+# Code markers configuration
+
+menu "Marker configuration"
+
+
+config MARK
+	bool "Enable MARK code markers"
+	default y
+	help
+	  Activate markers that can call printk or can be instrumented
+	  dynamically.
+
+choice
+	prompt "MARK code marker behavior"
+	default MARK_KPROBE
+	depends on MARK
+	help
+	  Configuration of markers that can call printk or can be
+	  instrumented dynamically.
+
+config MARK_KPROBE
+	bool "KPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event".
+config MARK_JPROBE
+	bool "JPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event"
+	and create a target for a high speed dynamic probe.
+config MARK_FPROBE
+	bool "FPROBE"
+	---help---
+	Change markers for a function call.
+config MARK_PRINT
+	bool "PRINT"
+	---help---
+	Call printk from the marker.
+endchoice
+
+config MARK_NOPRINT
+	bool "Enable MARK_NOPRINT code markers"
+	default y
+	help
+	  Activate markers that cannot call printk.
+ 
+choice
+	prompt "MARK_NOPRINT code marker behavior"
+	default MARK_NOPRINT_KPROBE
+	depends on MARK_NOPRINT
+	help
+	  Configuration of markers that cannot call printk.
+
+config MARK_NOPRINT_KPROBE
+	bool "KPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event".
+config MARK_NOPRINT_JPROBE
+	bool "JPROBE"
+	---help---
+	Change markers for a symbol "__mark_modulename_event"
+	and create a target for a high speed dynamic probe.
+config MARK_NOPRINT_FPROBE
+	bool "FPROBE"
+	---help---
+	Change markers for a function call.

kernel/Kconfig.marker line 75

Quoted 10 lines
kernel/Kconfig.marker new file +10 -0
+endchoice
+
+config MARK_STATIC
+	bool "Enable MARK_STATIC code markers"
+	default n
+	help
+	  Activate markers that cannot be instrumented dynamically. They will
+	  generate function calls to each function-style probe.
+
+endmenu

I think having so many config options is not a good idea, you can group them and reduce the number of config options.

Thanks
Prasanna

Signature · 5 lines
Prasanna S.P.
Linux Technology Center
India Software Labs, IBM Bangalore
Email: [email protected]
Ph: 91-80-41776329
#2 lore raw
In reply to Randy.Dunlap

Hi Randy,

Thanks for the comments, see below :

Quoted 8 lines · * Randy.Dunlap ([email protected]) wrote:

include/linux/marker.h line 91

Quoted 3 lines
include/linux/marker.h new file +3 -0
+static inline void __mark_check_format(const char *fmt, ...)
+	__attribute__ ((format (printf, 1, 2)));
+void __mark_check_format(const char *fmt, ...) {  }

That last line is confusing (to me). What's it for? Is it just an empty (inline) function definition? If so, why repeat the void + function name?

The goal of this "empty" function is just to have to compiler check the string format consistency.

I separated the function declaration and implementation because I have seen some compilers complain about having the two merged together.

I will change it to

static inline __attribute__ ((format (printf, 1, 2))) void __mark_check_format(const char *fmt, ...) { }

And hope every compiler will like it.

The empty implementation is because the function is called (must therefore be implemented), but I expect the compiler to completely optimize it away.

Quoted 5 lines
Quoted 5 lines
kernel/Kconfig.marker new file +2 -0
@@ -0,0 +1,75 @@
+# Code markers configuration
+
Quoted 9 lines

kernel/Kconfig.marker line 8

Quoted 6 lines
kernel/Kconfig.marker new file +6 -0
+menu "Marker configuration"
+
+
+config MARK
+	bool "Enable MARK code markers"
+	default y

Please justify using 'y' as the default value.

It has to be debated. The default for markers will put a symbol for all the markers, so that kprobe can easily attach to it. It has no impact that I am aware of except to boost the number of symbols.

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#5 lore raw
In reply to Mathieu Desnoyers
Quoted 4 lines · Mathieu Desnoyers

Following some very interesting ideas from Martin that shows that even static function calls and inlined functions can be used in interesting ways with markers to deploy dynamic tracers with easy access to local function variables [...]

While that has been an interesting idea, its implementation would need to be sketched out far beyond the speculative, in order to override what I perceive as the rough consensus reached during earlier in the thread.

Quoted 3 lines

[...] These last emails convince me even more that a markup mechanism must interface with every kind of instrumentation hooking we can think about, both dynamic and static. [...]

If you don't allow yourself to presume on-the-fly function recompilation, then these markers would need to be made run-time rather than compile-time configurable. That is, not like this:

include/linux/marker.h line 71

Quoted 12 lines
include/linux/marker.h new file +12 -0
+/* Menu configured markers */
+#ifndef CONFIG_MARK
+#define MARK	MARK_INACTIVE
+#elif defined(CONFIG_MARK_PRINT)
+#define MARK	MARK_PRINT
+#elif defined(CONFIG_MARK_FPROBE)
+#define MARK	MARK_FPROBE
+#elif defined(CONFIG_MARK_KPROBE)
+#define MARK	MARK_KPROBE
+#elif defined(CONFIG_MARK_JPROBE)
+#define MARK	MARK_JPROBE
+#endif

- FChE

#6 lore raw
Quoted 4 lines · * Frank Ch. Eigler ([email protected]) wrote:

If you don't allow yourself to presume on-the-fly function recompilation, then these markers would need to be made run-time rather than compile-time configurable. That is, not like this:

Quoted 12 lines

include/linux/marker.h line 71

Quoted 12 lines
include/linux/marker.h new file +12 -0
+/* Menu configured markers */
+#ifndef CONFIG_MARK
+#define MARK	MARK_INACTIVE
+#elif defined(CONFIG_MARK_PRINT)
+#define MARK	MARK_PRINT
+#elif defined(CONFIG_MARK_FPROBE)
+#define MARK	MARK_FPROBE
+#elif defined(CONFIG_MARK_KPROBE)
+#define MARK	MARK_KPROBE
+#elif defined(CONFIG_MARK_JPROBE)
+#define MARK	MARK_JPROBE
+#endif

By making them run-time configurable, I don't see any whay not to bloat the kernel. How can be embed calls to printk+function+kprobe+djprobe without having some kind of performance impact ?

Do you have any suggestion for this ? (or maybe am I missing your point ?)

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#7 lore raw

Hi -

Quoted 9 lines · Mathieu Desnoyers

[...]

Quoted 4 lines

If you don't allow yourself to presume on-the-fly function recompilation, then these markers would need to be made run-time rather than compile-time configurable. That is, not like this:
[...]

By making them run-time configurable, I don't see any whay not to bloat the kernel. How can be embed calls to printk+function+kprobe+djprobe without having some kind of performance impact ?

In order to have what we appear to need, we cannot avoid having some impact. (Even NOPs have impact.)

Suppose that mbligh's clever but speculative idea has some nasty flaw, once someone tried to reduce it to code. Do you see that markers along the lines you've posted would be unsatisfactory? With that in mind, is there point adding such markers now?

- FChE

#4 lore raw
In reply to S. P. Prasanna

Hi,

Quoted 3 lines · * S. P. Prasanna ([email protected]) wrote:

I think having so many config options is not a good idea, you can group them and reduce the number of config options.

Then we would have to determine what the scenarios are. The problem is to cover all interesting instrumentation mixes efficiently.

I think it could be a good enough list :

Fprobes only
Dynamic + Fprobes (supports dynamic probes and uses fprobes for non probable
code)
Dynamic only
Printk only

Which would be expressed in the following menu :

choice Marker behavior
* Inactive
* Dynamic probes
* Function probes (Fprobes)
* Dynamic probes complemented with Fprobes
* Printk

if selected "Dynamic probes" or "Dynamic probes complemented with Fprobes"
choice2 Dynamic probes behavior
* Kprobes
* Jprobes

Any thoughts ?

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#8 lore raw
In reply to Frank Ch. Eigler
Quoted 9 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 5 lines · Mathieu Desnoyers

[...]

Quoted 4 lines

If you don't allow yourself to presume on-the-fly function recompilation, then these markers would need to be made run-time rather than compile-time configurable. That is, not like this:
[...]

Quoted 7 lines
Quoted 3 lines

By making them run-time configurable, I don't see any whay not to bloat the kernel. How can be embed calls to printk+function+kprobe+djprobe without having some kind of performance impact ?

In order to have what we appear to need, we cannot avoid having some impact. (Even NOPs have impact.)

I am all for giving this decision to the end-user or the distribution which will configure the kernel. There is no "perfect" or "for all" solution that I am aware of.

* Users debugging servers will more likely want the kprobe or jprobe option.
* Users interested in high performance tracing will want fprobe and/or jprobe.
* Users interested in embedded systems will want to avoid tools outside the kernel that rely on module loading : their kernel often not even support modules. -> fprobe

Quoted 5 lines

Suppose that mbligh's clever but speculative idea has some nasty flaw, once someone tried to reduce it to code. Do you see that markers along the lines you've posted would be unsatisfactory? With that in mind, is there point adding such markers now?

M. Bligh's idea is an interesting use of fprobes through modules that could make dynamic tracing more effective for accessing local variables. It does not change anything to the various needs of the above-mentioned class of users, except that it may make life of high performance and server users easier. With or without his idea, the goal of this marker mechanism is to meet all those user's different needs.

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#9 lore raw

Hi -

Quoted 6 lines · Mathieu Desnoyers

[...]

Quoted 2 lines

In order to have what we appear to need, we cannot avoid having some impact. (Even NOPs have impact.)

I am all for giving this decision to the end-user or the distribution which will configure the kernel. [...]

If the decision you're talking about is whether all markers in the system should behave one way or another, then this is a degree of central control that we have not contemplated during the entire thread, until now.

It is an end-user such as an administrator who will figure out which probes/markers/tracing elements need what kind of processing attached. They don't want to recompile the kernel to switch. They will want different types of processing, or none at all, for different markers during a system lifetime.

Quoted 6 lines

* Users debugging servers will more likely want the kprobe or jprobe option.
* Users interested in high performance tracing will want fprobe and/or jprobe.
* Users interested in embedded systems will want to avoid tools outside the kernel that rely on module loading: their kernel often not even support modules. -> fprobe

This line of thinking makes me worry that we've forgotten all that we learned during the weekend. Amongst the insights apparently agreed was that on *any given system*, a mixture of static an dynamic probing was likely necessary. For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

(Regarding module loading being considered a blocker for a tool like systemtap, don't. We will support pre-compiled boot-time instrumentation loaded from e.g. initrd or linked right into vmlinux.)

Quoted 3 lines

M. Bligh's idea is an interesting use of fprobes through modules that could make dynamic tracing more effective for accessing local variables. [...]

That's if it works, if it can be implemented, if it does not create conflicts between multiple tracing/probing systems, if ...

Yes, in theory it might bridge the gulf between compile-time and run-time configuration, but aren't these all big "if"s right now?

Quoted 2 lines

With or without his idea, the goal of this marker mechanism is to meet all those user's different needs.

I don't understand how this new compile-time configured style of marker is to serve anyone who wants to use something other than a single distribution-picked tracing/probing tool. I though we had abandoned that model some time ago.

- FChE

#10 lore raw
Quoted 10 lines · Frank Ch. Eigler

If the decision you're talking about is whether all markers in the system should behave one way or another, then this is a degree of central control that we have not contemplated during the entire thread, until now.

It is an end-user such as an administrator who will figure out which probes/markers/tracing elements need what kind of processing attached. They don't want to recompile the kernel to switch. They will want different types of processing, or none at all, for different markers during a system lifetime.

Sure. I'm sure there's just a slight miscommunication here because my understanding of Mathieu's attempts goes in the direction of what you're saying here. He might not have gotten it right, but that can be worked on.

Quoted 6 lines

This line of thinking makes me worry that we've forgotten all that we learned during the weekend. Amongst the insights apparently agreed was that on *any given system*, a mixture of static an dynamic probing was likely necessary. For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

Ok. So if I get what you're saying here, you'd like to be able to overload a marker? Can you suggest a macro that can do what you'd like. I'm sure Mathieu would gladly take a close look at it.

Quoted 5 lines

That's if it works, if it can be implemented, if it does not create conflicts between multiple tracing/probing systems, if ...

Yes, in theory it might bridge the gulf between compile-time and run-time configuration, but aren't these all big "if"s right now?

Lots of "if"s in this thread, and this weekend does teach us that highlighting the problems with other peoples' "if"s is dangerous. I'm sure you'd agree that concentrating on the areas where there is agreement would be best.

Quoted 4 lines

I don't understand how this new compile-time configured style of marker is to serve anyone who wants to use something other than a single distribution-picked tracing/probing tool. I though we had abandoned that model some time ago.

We did, and I'm sure there's a fundamental misunderstanding here. It would likely help if you could give a concrete example of how you would like Mathieu's proposal be changed or, if you don't like it at all, what you would like to see. Anything purely technical that will avoid any of the pitfalls generated by differences in perspective.

Thanks,

Karim

#11 lore raw

Hi -

Quoted 6 lines
Quoted 3 lines

[...] For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

Ok. So if I get what you're saying here, you'd like to be able to overload a marker?

Sort of. Remember, we discussed markers as *marking* places and things, with the intent that they be decoupled from the actual *action* that is taken when the marker is hit.

Quoted 1 line

Can you suggest a macro that can do what you'd like. [...]

Compare the kind of marker I showed at OLS and presently supported by systemtap. Its unparametrized version looks like this:

#define STAP_MARK(name) do { \
static void (*__mark_##name##_)(); \
if (unlikely (__mark_##name##_)) \
(void) (__mark_##name##_()); \
} while (0)

A tracing/probing tool would hook up to a particular and specific marker at run time by locating the __mark_NAME static variable (a function pointer) in the data segment, for example using the ordinary symbol table, and swapping into it the address of a compatible back-end handler function. When a particular tracing/probing session ends, the function pointer is reset to null.

Note that this technique:

- operates at run time
- is portable
- in its parametrized variants, is type-safe
- does not require any future technology
- does impose some overhead even when a marker is not active

- FChE

#12 lore raw
Quoted 31 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 6 lines
Quoted 3 lines

[...] For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

Ok. So if I get what you're saying here, you'd like to be able to overload a marker?

Sort of. Remember, we discussed markers as *marking* places and things, with the intent that they be decoupled from the actual *action* that is taken when the marker is hit.

Quoted 1 line

Can you suggest a macro that can do what you'd like. [...]

Compare the kind of marker I showed at OLS and presently supported by systemtap. Its unparametrized version looks like this:

#define STAP_MARK(name) do { \
   static void (*__mark_##name##_)(); \
   if (unlikely (__mark_##name##_)) \
   (void) (__mark_##name##_()); \
} while (0)

A tracing/probing tool would hook up to a particular and specific marker at run time by locating the __mark_NAME static variable (a function pointer) in the data segment, for example using the ordinary symbol table, and swapping into it the address of a compatible back-end handler function. When a particular tracing/probing session ends, the function pointer is reset to null.

Quoted 9 lines

Note that this technique:

- operates at run time
- is portable
- in its parametrized variants, is type-safe
- does not require any future technology
- does impose some overhead even when a marker is not active

Hi Frank,

Yes, I think there is much to gain to switch from the 5 nops "jumpprobe" to this scheme. In its parametrized variant, the jump will probably jump over a stack setup and function call. Do you think I should simply switch from the 5 nops marker to this technique ? I guess the performance impact of a predicted branch will be similar to 5 nops anyway...

The clear advantage I see in the parametrized variant is that the parameters will be ready for the called function : it makes it trivial to access any variable from the traced function.

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#17 lore raw
In reply to Frank Ch. Eigler

Thinking about it, I would leave a choice of two options :

1 - the probe mechanism you described at OLS + passing arguments 2 - a simple symbol for kprobe

Any thoughts ?

Mathieu

Quoted 33 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 6 lines
Quoted 3 lines

[...] For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

Ok. So if I get what you're saying here, you'd like to be able to overload a marker?

Sort of. Remember, we discussed markers as *marking* places and things, with the intent that they be decoupled from the actual *action* that is taken when the marker is hit.

Quoted 1 line

Can you suggest a macro that can do what you'd like. [...]

Compare the kind of marker I showed at OLS and presently supported by systemtap. Its unparametrized version looks like this:

#define STAP_MARK(name) do { \
   static void (*__mark_##name##_)(); \
   if (unlikely (__mark_##name##_)) \
   (void) (__mark_##name##_()); \
} while (0)

A tracing/probing tool would hook up to a particular and specific marker at run time by locating the __mark_NAME static variable (a function pointer) in the data segment, for example using the ordinary symbol table, and swapping into it the address of a compatible back-end handler function. When a particular tracing/probing session ends, the function pointer is reset to null.

Note that this technique:

Quoted 8 lines

- operates at run time
- is portable
- in its parametrized variants, is type-safe
- does not require any future technology
- does impose some overhead even when a marker is not active

- FChE

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#13 lore raw
In reply to Mathieu Desnoyers

Hi Frank,

Here is a revised proposal (just the marker.h). Do you have ideas on how we can export the function symbol ? (is it necessary ?)

Any thoughts ?

----- BEGIN -----

#include <asm/marker.h>

#ifdef CONFIG_MARK_SYMBOL

#define MARK_SYM(name) \
        do  { \
                __asm__ ( "__mark_kprobe_" #name ":" ); \
        } while(0)

#else
#define MARK_SYM(name)
#endif

#ifdef CONFIG_MARK_CALL

#define MARK_CALL(name, format, args...) \
        do {\
                static void (*__mark_call_##name##_)(const char *fmt, ...) \
                        asm ("__mark_call_"#name); \
                if (unlikely (__mark_call_##name##_)) \
                        (void) (__mark_call_##name##_(format, ## args)); \
        } while(0)

#else
#define MARK_CALL(name, format, args...)
#endif

#define MARK(name, format, args...) \
        do { \
                __mark_check_format(format, ## args); \
                MARK_SYM(name); \
                MARK_CALL(name, format, ## args); \
        } while(0)

static inline __attribute__ ((format (printf, 1, 2))) void __mark_check_format(const char *fmt, ...) { }

---- END ----

Quoted 53 lines · * Mathieu Desnoyers ([email protected]) wrote:
Quoted 31 lines · * Frank Ch. Eigler ([email protected]) wrote:

Hi -

Quoted 6 lines
Quoted 3 lines

[...] For the static part of the instrumentation, a marker that could be hooked up to either type of probing system was desirable, which implies some sort of run-time changeability.

Ok. So if I get what you're saying here, you'd like to be able to overload a marker?

Sort of. Remember, we discussed markers as *marking* places and things, with the intent that they be decoupled from the actual *action* that is taken when the marker is hit.

Quoted 1 line

Can you suggest a macro that can do what you'd like. [...]

Compare the kind of marker I showed at OLS and presently supported by systemtap. Its unparametrized version looks like this:

#define STAP_MARK(name) do { \
   static void (*__mark_##name##_)(); \
   if (unlikely (__mark_##name##_)) \
   (void) (__mark_##name##_()); \
} while (0)

A tracing/probing tool would hook up to a particular and specific marker at run time by locating the __mark_NAME static variable (a function pointer) in the data segment, for example using the ordinary symbol table, and swapping into it the address of a compatible back-end handler function. When a particular tracing/probing session ends, the function pointer is reset to null.

Quoted 9 lines

Note that this technique:

- operates at run time
- is portable
- in its parametrized variants, is type-safe
- does not require any future technology
- does impose some overhead even when a marker is not active

Hi Frank,

Yes, I think there is much to gain to switch from the 5 nops "jumpprobe" to this scheme. In its parametrized variant, the jump will probably jump over a stack setup and function call. Do you think I should simply switch from the 5 nops marker to this technique ? I guess the performance impact of a predicted branch will be similar to 5 nops anyway...

The clear advantage I see in the parametrized variant is that the parameters will be ready for the called function : it makes it trivial to access any variable from the traced function.

Quoted 5 lines

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#14 lore raw

Hi -

Quoted 2 lines · Mathieu Desnoyers

[...] Do you have ideas on how we can export the function symbol? (is it necessary ?)

It turns out that static variables like that get included in the ordinary symbol tables along with other (un)initialized globals - it has been making it into /proc/kallsyms. If the normal symbol table is not available, then some other measure would be needed to find the variable containing the function pointer.

Quoted 7 lines

[...]

#define MARK(name, format, args...) \
        do { \
                __mark_check_format(format, ## args); \
                MARK_SYM(name); \
                MARK_CALL(name, format, ## args); \
        } while(0)

While varargs simplify some things, it sacrifices type-safety, in that a handler function would have to be varargs too. For the systemtap marker prototype, parametrized variants use scores of (automatically generated) macros, with different arity/type permutations, each self-describing and type-safe.

Regarding a marker variant that would require kprobes (inserting a labelled NOP or few), it may be an appropriate choice where dormant marker overhead must be minimal and robust parameter passing is less important.

- FChE

#15 lore raw
Quoted 6 lines · * Frank Ch. Eigler ([email protected]) wrote:

While varargs simplify some things, it sacrifices type-safety, in that a handler function would have to be varargs too. For the systemtap marker prototype, parametrized variants use scores of (automatically generated) macros, with different arity/type permutations, each self-describing and type-safe.

The format string could be used to provide some kind of type safety : the compiler will check that arguments match the format string provided. From there, a simple script can parse the format string and generate a function prototype accordingly. Correct me if I am wrong, but I think that if the called function has the exact same parameter layout as the varargs caller stack, the function call should work (without the called function having a variable arguments list).

Quoted 5 lines

Regarding a marker variant that would require kprobes (inserting a labelled NOP or few), it may be an appropriate choice where dormant marker overhead must be minimal and robust parameter passing is less important.

I even came with the following idea :

Instead of using a test + conditional predicted branch, we could jump to an address locate just after the probe.

jmp to over_symbol address
call_symbol
call function pointer
over_symbol

This way, we could have portable :
- direct inconditional jump to an address following the marked site when disabled
- Enable stack setup and function call by setting the function pointer and changing the jmp target to be "call_symbol"
- Enable "direct jump to arbitrary assembly" by setting the jump target to arbitrary code, where this code can end by jumping to over_symbol.

The generated binary on x86 looks like :

  10:   a1 24 00 00 00          mov    0x24,%eax
  15:   ff e0                   jmp    *%eax
  17:   c7 44 24 04 01 00 00    movl   $0x1,0x4(%esp)
  1e:   00 
  1f:   c7 04 24 00 00 00 00    movl   $0x0,(%esp)
  26:   ff 15 1c 00 00 00       call   *0x1c

With those symbols :

f8875c08 b __mark_subsys_mark1_call [test_mark] (function pointer)
f8875620 d __mark_subsys_mark1_jump_call [test_mark]
f8875624 d __mark_subsys_mark1_jump_over [test_mark]

The macro doing that :

#define MARK_CALL(name, format, args...) \
        do {\
                __label__ call_label, over_label; \
                static void *__mark_##name##_jump_over \
                        asm ("__mark_"#name"_jump_over") = \
                        &&over_label; \
                static void *__mark_##name##_jump_call \
                                asm ("__mark_"#name"_jump_call") \
                                __attribute__((unused)) =  \
                                &&call_label; \
                static void (*__mark_##name##_call)(const char *fmt, ...) \
                        asm ("__mark_"#name"_call") = __mark_empty_function; \
                goto *__mark_##name##_jump_over; \
call_label: \
                (void) (__mark_##name##_call(format, ## args)); \
over_label: \
                do {} while(0); \
        } while(0)

A problem I saw in your approach was that there was no way to remove the function pointer without taking the risk to break everything.

The solution I came up with is to set the function to an empty __mark_empty_function when disabled, and set another function pointer to enable it.

Any thoughts ?

Mathieu

OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
#16 lore raw
In reply to Frank Ch. Eigler

And here is the sample module to use my jump-marker symbols :

(yes, it works!)

Adresses are taken by hand from /proc/kallsyms for now.

---BEGIN---

/* test-mark.c
*
*/

#include <linux/marker.h>
#include <linux/module.h>

static void **__mark_subsys_mark1_call = (void**)0xf887580c; static void **__mark_subsys_mark1_jump_over = (void**)0xf8875814; static void **__mark_subsys_mark1_jump_call = (void*)0xf8875810; static void *__this_mark_empty_function = (void*)0xf8875000;

static void *saved_over;

void do_mark1(const char *format, int value)

{
	printk("value is %d\n", value);
}

int init_module(void)

{
	*__mark_subsys_mark1_call = (void*)do_mark1;
	saved_over = *__mark_subsys_mark1_jump_over;
	*__mark_subsys_mark1_jump_over = *__mark_subsys_mark1_jump_call;

	return 0;
}

void cleanup_module(void)

{
	*__mark_subsys_mark1_jump_over = saved_over;
	*__mark_subsys_mark1_call = __this_mark_empty_function;
}

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Mathieu Desnoyers");
MODULE_DESCRIPTION("Marker Test");

---END---

OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68