support of lock profiling in Xen
PATCH 1/1 support of lock profiling in Xen
Hi,
attached is a patch to support lock profiling in the Xen hypervisor. The feature is switchable at compile time (like software performance counters) in Rules.mk. If lock profiling is not activated, code and data structures are the same as without the patch. With activated lock profiling some statistical data is added to each spinlock_t structure and locking/unlocking keeps track for each lock of:
- number of successful locks
- cumulated time of lock held
- number of locking calls leading to wait cycles
- cumulated waiting time
The profile data may be obtained either via hotkey from the console or with the new tool xenlockprof.
To be able to find all locks for printing out the profile data, a linked list with the locks is built up. Locks defined via DEFINE_SPINLOCK are added to this list automatically, locks in dynamically allocated data structures have to be added (and removed) explicitly to the queue to be able to print the profile data.
Adding is done via spin_lock_init_prof (there are two additional calls with one or two index parameters to be able to distinguish locks e.g. per domid and vcpu-nr), removing is done via spin_lock_destroy_prof (may be called for a completely zeroed structure as well).
I'm not completely satisfied with the solution for the dynamically initialized locks, but I had no better idea in the first run.
Another enhancement would be to expand the profiling to rw-locks as well, but this would have required a rewrite of the lock routines using try_lock like for spinlocks. I could do this if the lock profiling is accepted.
Comments welcome :-)
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Support of lock profiling in Xen
tools/libxc/xc_misc.c +24 -0
@@ -121,6 +121,30 @@return rc; } +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data) +{ +int rc; +DECLARE_SYSCTL ; + + sysctl.cmd = XEN_SYSCTL_lockprof_op; + sysctl.u.lockprof_op.cmd = opcode; + sysctl.u.lockprof_op.max_elem = n_elems ? *n_elems :0 ; +set_xen_guest_handle (sysctl.u.lockprof_op.data, data); + + rc =do_sysctl (xc_handle, &sysctl); + +if (n_elems) + *n_elems = sysctl.u.lockprof_op.nr_elem; +if (time) + *time = sysctl.u.lockprof_op.time; + +return rc; +} +int xc_getcpuinfo (int xc_handle,int max_cpus,xc_cpuinfo_t *info,int *nr_cpus) {
tools/libxc/xenctrl.h +8 -0
@@ -700,6 +700,14 @@int *nbr_desc, int *nbr_val); +typedef xen_sysctl_lockprof_data_t xc_lockprof_data_t ; +/* IMPORTANT: The caller is responsible for mlock()'ing the @data array. */ +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data); +/** * Memory maps a range within one domain to a local address range. Mappings * should be unmapped with munmap and should follow the same rules as mmap
tools/misc/Makefile +3 -3
@@ -10,7 +10,7 @@ HDRS = $(wildcard *.h) -TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat +TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx TARGETS := $(TARGETS-y)
@@ -22,7 +22,7 @@ INSTALL_BIN-$(CONFIG_X86) += xen-detect INSTALL_BIN := $(INSTALL_BIN-y) -INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat +INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx INSTALL_SBIN := $(INSTALL_SBIN-y)
@@ -49,7 +49,7 @@ %.o: %.c $(HDRS) Makefile $(CC) -c $(CFLAGS) -o $@ $< -xen-hvmctx xenperf xenpm gtracestat: %: %.o Makefile +xen-hvmctx xenperf xenpm gtracestat xenlockprof: %: %.o Makefile $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LDFLAGS_libxenctrl) gtraceview: %: %.o Makefile
tools/misc/xenlockprof.c new file +129 -0
129 lines not on this page · raw message
xen/Rules.mk +2 -0
@@ -6,6 +6,7 @@ verbose ?= n perfc ?= n perfc_arrays ?= n +lock_profile ?= n crash_debug ?= n frame_pointer ?= n
@@ -49,6 +50,7 @@ CFLAGS-$(crash_debug) += -DCRASH_DEBUG CFLAGS-$(perfc) += -DPERF_COUNTERS CFLAGS-$(perfc_arrays) += -DPERF_ARRAYS +CFLAGS-$(lock_profile) += -DLOCK_PROFILE CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER ifneq ($(max_phys_cpus),)
xen/arch/x86/xen.lds.S +7 -0
@@ -62,6 +62,13 @@ *(.data.read_mostly) } :text +#ifdef LOCK_PROFILE + . = ALIGN(32); + __lock_profile_start = .; + .lockprofile.data : { *(.lockprofile.data) } :text + __lock_profile_end = .; +#endif + . = ALIGN(4096); /* Init code and data */ __init_begin = .; .init.text : {
xen/common/keyhandler.c +20 -0
@@ -347,6 +347,20 @@ };#endif +#ifdef LOCK_PROFILE +extern void spinlock_profile_printall (unsigned char key); +static struct keyhandler spinlock_printall_keyhandler = { + .diagnostic =1 , + .u.fn = spinlock_profile_printall, + .desc ="print lock profile info" +}; +extern void spinlock_profile_reset (unsigned char key); +static struct keyhandler spinlock_reset_keyhandler = { + .u.fn = spinlock_profile_reset, + .desc ="reset lock profile info" +}; +#endif +static void run_all_nonirq_keyhandlers (unsigned long unused) {/* Fire all the non-IRQ-context diagnostic keyhandlers */
@@ -428,6 +442,12 @@register_keyhandler ('p' , &perfc_printall_keyhandler);register_keyhandler ('P' , &perfc_reset_keyhandler);#endif + +#ifdef LOCK_PROFILE +register_keyhandler ('l' , &spinlock_printall_keyhandler); +register_keyhandler ('L' , &spinlock_reset_keyhandler); +#endif + }/*
xen/common/spinlock.c +200 -0
279 lines not on this page · raw message
xen/common/sysctl.c +12 -0
@@ -29,6 +29,9 @@extern long arch_do_sysctl (struct xen_sysctl *op,XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl); +#ifdef LOCK_PROFILE +extern int spinlock_profile_control (xen_sysctl_lockprof_op_t *pc); +#endif long do_sysctl (XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl) {
@@ -144,6 +147,15 @@break ;#endif +#ifdef LOCK_PROFILE +case XEN_SYSCTL_lockprof_op: + { + ret =spinlock_profile_control (&op->u.lockprof_op); +if (copy_to_guest (u_sysctl, op,1 ) ) + ret = -EFAULT ; + } +break ; +#endif case XEN_SYSCTL_debug_keys: {char c;
xen/include/public/sysctl.h +29 -0
@@ -454,6 +454,34 @@#define PG_OFFLINE_OWNER_SHIFT 16 +#define XEN_SYSCTL_lockprof_op 11 +/* Sub-operations: */ +#define XEN_SYSCTL_LOCKPROF_reset 1/* Reset all profile data to zero. */ +#define XEN_SYSCTL_LOCKPROF_query 2/* Get lock profile information. */ +struct xen_sysctl_lockprof_data { +char name[40 ];/* lock name (may include up to 2 %d specifiers) */ +int32_t idx1;/* index 1 (e.g. domain id) or -1 */ +int32_t idx2;/* index 2 (e.g. vcpu nr) or -1 */ +uint64_t lock_cnt;/* # of locking succeeded */ +uint64_t block_cnt;/* # of wait for lock */ +uint64_t lock_time;/* nsecs lock held */ +uint64_t block_time;/* nsecs waited for lock */ +}; +typedef struct xen_sysctl_lockprof_data xen_sysctl_lockprof_data_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_data_t); +struct xen_sysctl_lockprof_op { +/* IN variables. */ +uint32_t cmd;/* XEN_SYSCTL_LOCKPROF_??? */ +uint32_t max_elem;/* size of output buffer */ +/* OUT variables (query only). */ +uint32_t nr_elem;/* number of elements available */ +uint64_t time;/* nsecs of profile measurement */ +/* profile information (or NULL) */ +XEN_GUEST_HANDLE_64 (xen_sysctl_lockprof_data_t) data; +}; +typedef struct xen_sysctl_lockprof_op xen_sysctl_lockprof_op_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_op_t); +struct xen_sysctl {uint32_t cmd;uint32_t interface_version;/* XEN_SYSCTL_INTERFACE_VERSION */
@@ -471,6 +499,7 @@struct xen_sysctl_cpu_hotplug cpu_hotplug;struct xen_sysctl_pm_op pm_op;struct xen_sysctl_page_offline_op page_offline; +struct xen_sysctl_lockprof_op lockprof_op;uint8_t pad[128 ]; } u; };
xen/include/xen/spinlock.h +48 -2
65 lines not on this page · raw message
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 7 lines · Juergen Gross
I'm not completely satisfied with the solution for the dynamically initialized locks, but I had no better idea in the first run.
Another enhancement would be to expand the profiling to rw-locks as well, but this would have required a rewrite of the lock routines usingtry_locklike for spinlocks. I could do this if the lock profiling is accepted.Comments welcome :-)
The method of chaining and initialising the info is kind of icky. Requiring users to unchain dynamic locks is just asking for this support to be perpetually broken, or be used only for static locks.
How about defining new initialisers DEFINE_NAMED_SPINLOCK() and named_spinlock_init(). This would indicate you consider a lock important enough to name (and hence profile) and also categorise dynamically-allocated locks, causing their stats to be aggregated (after all, lock optimisations will have aggregate effect across all locks of that category).
If lock profiling is compiled in, have a static array of lock-profile descriptors (keeps things simple - could make it a growable array, or something). On lock init, walk the array looking for that name. If found, write the entry index into a new field in the spinlock struct. If not found, allocate next entry in lock-profile array, init it, and write its index into the spinlock struct.
On lock operations, if the index field in the spinlock is non-zero, update stats in the associated profile structure. Regarding races from locks aliasing to the same profile structure -- either assume that doesn't matter much, or perhaps update fields with atomic ops.
-- Keir
Quoted 1 line · Juergen Gross
Comments welcome :-)
There's also the question of whether this really gets you useful extra info than can be digested from a xentrace run. I suppose at least the info is easier to understand, and not many people know how to drive xentrace.
-- Keir
Quoted 20 lines · Keir Fraser
Quoted 7 lines · Juergen Gross
I'm not completely satisfied with the solution for the dynamically initialized locks, but I had no better idea in the first run.
Another enhancement would be to expand the profiling to rw-locks as well, but this would have required a rewrite of the lock routines usingtry_locklike for spinlocks. I could do this if the lock profiling is accepted.Comments welcome :-)
The method of chaining and initialising the info is kind of icky. Requiring users to unchain dynamic locks is just asking for this support to be perpetually broken, or be used only for static locks.
How about defining new initialisers
DEFINE_NAMED_SPINLOCK()andnamed_spinlock_init(). This would indicate you consider a lock important enough to name (and hence profile) and also categorise dynamically-allocated locks, causing their stats to be aggregated (after all, lock optimisations will have aggregate effect across all locks of that category).
Quoted 6 lines
If lock profiling is compiled in, have a static array of lock-profile descriptors (keeps things simple - could make it a growable array, or something). On lock init, walk the array looking for that name. If found, write the entry index into a new field in the spinlock struct. If not found, allocate next entry in lock-profile array, init it, and write its index into the spinlock struct.
Don't you think a solution like for performance counters would be better then? The index would be in a header file and the sums could be held per-cpu to avoid races.
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
Quoted 5 lines
On lock operations, if the index field in the spinlock is non-zero, update stats in the associated profile structure. Regarding races from locks aliasing to the same profile structure -- either assume that doesn't matter much, or perhaps update fields with atomic ops.
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Quoted 7 lines · Juergen Gross
Quoted 6 lines
If lock profiling is compiled in, have a static array of lock-profile descriptors (keeps things simple - could make it a growable array, or something). On lock init, walk the array looking for that name. If found, write the entry index into a new field in the spinlock struct. If not found, allocate next entry in lock-profile array, init it, and write its index into the spinlock struct.
Quoted 5 lines
Don't you think a solution like for performance counters would be better then? The index would be in a header file and the sums could be held per-cpu to avoid races.
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
It's just a question of impact on our existing locking API. I'd like to minimise that. I'm not particularly a fan of the perfc stuff.
-- Keir
Quoted 7 lines · Keir Fraser
Quoted 1 line · Juergen Gross
Comments welcome :-)
There's also the question of whether this really gets you useful extra info than can be digested from a xentrace run. I suppose at least the info is easier to understand, and not many people know how to drive xentrace.
It helped me a lot for narrowing down my performance problem with a 8 vcpu BS2000 system (this was the main reason I made the patch). Finding such a problem in xentrace is quite a bit of work. xentrace produced over 500 MB of data in 30 seconds, while reading the xenlockprof output revealed the bottleneck in seconds.
And I think the lock profiling tool is much better for evaluating optimizations in Xen regarding lock conflicts, if the differences are only a few percent.
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Quoted 2 lines · Juergen Gross
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
Allowing spinlock_init() to specify not only a name but also a 'category-specific index' might be okay. For example, domid, in the case of a domain-specific lock. Hiding the allocation of profiling info structs in the spinlock subsystem somehow... (e.g., radix tree to make a simple growable array).
-- Keir
Quoted 10 lines · Keir Fraser
Quoted 2 lines · Juergen Gross
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
Allowing
spinlock_init()to specify not only a name but also a 'category-specific index' might be okay. For example, domid, in the case of a domain-specific lock. Hiding the allocation of profiling info structs in the spinlock subsystem somehow... (e.g., radix tree to make a simple growable array).
I'll try another patch...
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Quoted 10 lines · Juergen Gross
Quoted 3 lines
There's also the question of whether this really gets you useful extra info than can be digested from a xentrace run. I suppose at least the info is easier to understand, and not many people know how to drive xentrace.
It helped me a lot for narrowing down my performance problem with a 8 vcpu BS2000 system (this was the main reason I made the patch). Finding such a problem in xentrace is quite a bit of work. xentrace produced over 500 MB of data in 30 seconds, while reading the xenlockprof output revealed the bottleneck in seconds.
Quoted 3 lines
And I think the lock profiling tool is much better for evaluating optimizations in Xen regarding lock conflicts, if the differences are only a few percent.
Okay, well the only thing I'm dead against in a lockmeter patch is requring dynamic registration and unregistration of individual locks. Largely because I see a never-ending stream of fix this unregistration, fix that unregistration, stretching out before me, because callers will forever be getting broken.
I suppose most locks are either global or per-domain, so perhaps you could just have domain create/destroy hooks into lockmeter, to alloc/free an array of per-domain locks? Would that actually be an argument for a perfc-style central declaration of all profileable locks, so you could indicate which are global and which are per-domain?
-- Keir
Quoted 10 lines · Juergen Gross
Quoted 8 lines
Quoted 2 lines
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
Allowing
spinlock_init()to specify not only a name but also a 'category-specific index' might be okay. For example, domid, in the case of a domain-specific lock. Hiding the allocation of profiling info structs in the spinlock subsystem somehow... (e.g., radix tree to make a simple growable array).I'll try another patch...
Well, see my email just now. I could perhaps live with perfc-style declarations after all... Because this might make sense if locks can be broadly categorised as global or as per-domain, and it could avoid you having to make your implementation much more complex?
After all, now I think about it, an 'arbitrary' index specified to spinlock_init() is just making our lives more complicated if it will basically *always* be a domid. And knowing at a higher level which locks are actually per-domain could help your pretty printing?
-- Keir
Quoted 16 lines · Keir Fraser
Quoted 8 lines · Juergen Gross
Quoted 7 lines
Quoted 2 lines
It would even be possible to define lock arrays if summing up all data for e.g. domain specific locks is not desired.
Allowing
spinlock_init()to specify not only a name but also a 'category-specific index' might be okay. For example, domid, in the case of a domain-specific lock. Hiding the allocation of profiling info structs in the spinlock subsystem somehow... (e.g., radix tree to make a simple growable array).I'll try another patch...
Well, see my email just now. I could perhaps live with perfc-style declarations after all... Because this might make sense if locks can be broadly categorised as global or as per-domain, and it could avoid you having to make your implementation much more complex?
Quoted 4 lines
After all, now I think about it, an 'arbitrary' index specified to
spinlock_init()is just making our lives more complicated if it will basically *always* be a domid. And knowing at a higher level which locks are actually per-domain could help your pretty printing?
I thought in this direction, too.
I would, however, try not to limit it to per domain locks, even if the first implementation would support only global locks and per domain locks.
The perfc style declarations should only be necessary for the dynamically initialized locks, and those could live near the domain structure definition (or any other structure where they are needed).
I think the statically defined locks can be handled via an own data section requiring no centrally defined index.
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Quoted 9 lines · Juergen Gross
Quoted 4 lines
After all, now I think about it, an 'arbitrary' index specified to
spinlock_init()is just making our lives more complicated if it will basically *always* be a domid. And knowing at a higher level which locks are actually per-domain could help your pretty printing?I thought in this direction, too.
I would, however, try not to limit it to per domain locks, even if the first implementation would support only global locks and per domain locks.
Quoted 6 lines
The perfc style declarations should only be necessary for the dynamically initialized locks, and those could live near the domain structure definition (or any other structure where they are needed).
I think the statically defined locks can be handled via an own data section requiring no centrally defined index.
Okay, well this all sounds an acceptable kind of direction. Worth spinning another patch.
-- Keir
Quoted 14 lines · Keir Fraser
Quoted 12 lines · Juergen Gross
Quoted 4 lines
After all, now I think about it, an 'arbitrary' index specified to
spinlock_init()is just making our lives more complicated if it will basically *always* be a domid. And knowing at a higher level which locks are actually per-domain could help your pretty printing?I thought in this direction, too.
I would, however, try not to limit it to per domain locks, even if the first implementation would support only global locks and per domain locks.The perfc style declarations should only be necessary for the dynamically initialized locks, and those could live near the domain structure definition (or any other structure where they are needed).
Quoted 5 lines
Quoted 2 lines
I think the statically defined locks can be handled via an own data section requiring no centrally defined index.
Okay, well this all sounds an acceptable kind of direction. Worth spinning another patch.
Here it is.
I managed to avoid the perfc-like stuff completely.
Included are 2 global locks defined for profiling and 2 locks in the domain structure.
I've added some comments in spinlock.h how to use the profiling in structures.
Tested by compilation and boot of Dom0 with and without profiling enabled.
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
Support of lock profiling
tools/libxc/xc_misc.c +24 -0
@@ -121,6 +121,30 @@return rc; } +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data) +{ +int rc; +DECLARE_SYSCTL ; + + sysctl.cmd = XEN_SYSCTL_lockprof_op; + sysctl.u.lockprof_op.cmd = opcode; + sysctl.u.lockprof_op.max_elem = n_elems ? *n_elems :0 ; +set_xen_guest_handle (sysctl.u.lockprof_op.data, data); + + rc =do_sysctl (xc_handle, &sysctl); + +if (n_elems) + *n_elems = sysctl.u.lockprof_op.nr_elem; +if (time) + *time = sysctl.u.lockprof_op.time; + +return rc; +} +int xc_getcpuinfo (int xc_handle,int max_cpus,xc_cpuinfo_t *info,int *nr_cpus) {
tools/libxc/xenctrl.h +8 -0
@@ -700,6 +700,14 @@int *nbr_desc, int *nbr_val); +typedef xen_sysctl_lockprof_data_t xc_lockprof_data_t ; +/* IMPORTANT: The caller is responsible for mlock()'ing the @data array. */ +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data); +/** * Memory maps a range within one domain to a local address range. Mappings * should be unmapped with munmap and should follow the same rules as mmap
tools/misc/Makefile +3 -3
@@ -10,7 +10,7 @@ HDRS = $(wildcard *.h) -TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat +TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx TARGETS := $(TARGETS-y)
@@ -22,7 +22,7 @@ INSTALL_BIN-$(CONFIG_X86) += xen-detect INSTALL_BIN := $(INSTALL_BIN-y) -INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat +INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx INSTALL_SBIN := $(INSTALL_SBIN-y)
@@ -49,7 +49,7 @@ %.o: %.c $(HDRS) Makefile $(CC) -c $(CFLAGS) -o $@ $< -xen-hvmctx xenperf xenpm gtracestat: %: %.o Makefile +xen-hvmctx xenperf xenpm gtracestat xenlockprof: %: %.o Makefile $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LDFLAGS_libxenctrl) gtraceview: %: %.o Makefile
tools/misc/xenlockprof.c new file +136 -0
136 lines not on this page · raw message
xen/Rules.mk +2 -0
@@ -6,6 +6,7 @@ verbose ?= n perfc ?= n perfc_arrays ?= n +lock_profile ?= n crash_debug ?= n frame_pointer ?= n
@@ -49,6 +50,7 @@ CFLAGS-$(crash_debug) += -DCRASH_DEBUG CFLAGS-$(perfc) += -DPERF_COUNTERS CFLAGS-$(perfc_arrays) += -DPERF_ARRAYS +CFLAGS-$(lock_profile) += -DLOCK_PROFILE CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER ifneq ($(max_phys_cpus),)
xen/arch/x86/pci.c +1 -1
@@ -8,7 +8,7 @@#include <xen/pci.h> #include <asm/io.h> -static DEFINE_SPINLOCK (pci_config_lock); +DEFINE_SPINLOCK_STATIC_PROF (pci_config_lock);uint32_t pci_conf_read (uint32_t cf8,uint8_t offset,uint8_t bytes) {
xen/arch/x86/smp.c +1 -1
@@ -201,7 +201,7 @@local_irq_restore (flags); } -static DEFINE_SPINLOCK (flush_lock); +DEFINE_SPINLOCK_STATIC_PROF (flush_lock);static cpumask_t flush_cpumask;static const void *flush_va;static unsigned int flush_flags;
xen/arch/x86/xen.lds.S +7 -0
@@ -62,6 +62,13 @@ *(.data.read_mostly) } :text +#ifdef LOCK_PROFILE + . = ALIGN(32); + __lock_profile_start = .; + .lockprofile.data : { *(.lockprofile.data) } :text + __lock_profile_end = .; +#endif + . = ALIGN(4096); /* Init code and data */ __init_begin = .; .init.text : {
xen/common/domain.c +25 -2
@@ -226,8 +226,8 @@ init_status |= INIT_xsm;atomic_set (&d->refcnt,1 ); -spin_lock_init (&d->domain_lock); -spin_lock_init (&d->page_alloc_lock); +spin_lock_init_prof (d, domain_lock); +spin_lock_init_prof (d, page_alloc_lock);spin_lock_init (&d->shutdown_lock);spin_lock_init (&d->hypercall_deadlock_mutex);INIT_PAGE_LIST_HEAD (&d->page_list);
@@ -884,6 +884,29 @@return -ENOSYS ; } +#ifdef LOCK_PROFILE +void domain_lock_prof_func (lock_profile_subfunc *sub,void *par) +{ +struct domain *d; + +rcu_read_lock (&domlist_read_lock); +for_each_domain (d) + { +sub ((struct lock_profile_data_array *)(&d->lock_profile_array), +LOCKPROF_TYPE_PERDOM ,"domain" , d->domain_id, par); + } +rcu_read_unlock (&domlist_read_lock); +return ; +} + +static int __init domain_lock_prof_init (void ) +{ + lock_profile_funcs[LOCKPROF_TYPE_PERDOM ] = domain_lock_prof_func; +return 0 ; +} +__initcall (domain_lock_prof_init); +#endif +/* * Local variables: * mode: C
xen/common/keyhandler.c +20 -0
@@ -347,6 +347,20 @@ };#endif +#ifdef LOCK_PROFILE +extern void spinlock_profile_printall (unsigned char key); +static struct keyhandler spinlock_printall_keyhandler = { + .diagnostic =1 , + .u.fn = spinlock_profile_printall, + .desc ="print lock profile info" +}; +extern void spinlock_profile_reset (unsigned char key); +static struct keyhandler spinlock_reset_keyhandler = { + .u.fn = spinlock_profile_reset, + .desc ="reset lock profile info" +}; +#endif +static void run_all_nonirq_keyhandlers (unsigned long unused) {/* Fire all the non-IRQ-context diagnostic keyhandlers */
@@ -428,6 +442,12 @@register_keyhandler ('p' , &perfc_printall_keyhandler);register_keyhandler ('P' , &perfc_reset_keyhandler);#endif + +#ifdef LOCK_PROFILE +register_keyhandler ('l' , &spinlock_printall_keyhandler); +register_keyhandler ('L' , &spinlock_reset_keyhandler); +#endif + }/*
xen/common/spinlock.c +260 -0
339 lines not on this page · raw message
xen/common/sysctl.c +12 -0
@@ -29,6 +29,9 @@extern long arch_do_sysctl (struct xen_sysctl *op,XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl); +#ifdef LOCK_PROFILE +extern int spinlock_profile_control (xen_sysctl_lockprof_op_t *pc); +#endif long do_sysctl (XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl) {
@@ -144,6 +147,15 @@break ;#endif +#ifdef LOCK_PROFILE +case XEN_SYSCTL_lockprof_op: + { + ret =spinlock_profile_control (&op->u.lockprof_op); +if (copy_to_guest (u_sysctl, op,1 ) ) + ret = -EFAULT ; + } +break ; +#endif case XEN_SYSCTL_debug_keys: {char c;
xen/include/public/sysctl.h +33 -0
@@ -454,6 +454,38 @@#define PG_OFFLINE_OWNER_SHIFT 16 +#define XEN_SYSCTL_lockprof_op 11 +/* Sub-operations: */ +#define XEN_SYSCTL_LOCKPROF_reset 1/* Reset all profile data to zero. */ +#define XEN_SYSCTL_LOCKPROF_query 2/* Get lock profile information. */ +/* Record-type: */ +#define LOCKPROF_TYPE_GLOBAL 0/* global lock, idx meaningless */ +#define LOCKPROF_TYPE_PERDOM 1/* per-domain lock, idx is domid */ +#define LOCKPROF_TYPE_N 2/* number of types */ +struct xen_sysctl_lockprof_data { +char name[40 ];/* lock name (may include up to 2 %d specifiers) */ +int32_t type;/* LOCKPROF_TYPE_??? */ +int32_t idx;/* index (e.g. domain id) */ +uint64_t lock_cnt;/* # of locking succeeded */ +uint64_t block_cnt;/* # of wait for lock */ +uint64_t lock_time;/* nsecs lock held */ +uint64_t block_time;/* nsecs waited for lock */ +}; +typedef struct xen_sysctl_lockprof_data xen_sysctl_lockprof_data_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_data_t); +struct xen_sysctl_lockprof_op { +/* IN variables. */ +uint32_t cmd;/* XEN_SYSCTL_LOCKPROF_??? */ +uint32_t max_elem;/* size of output buffer */ +/* OUT variables (query only). */ +uint32_t nr_elem;/* number of elements available */ +uint64_t time;/* nsecs of profile measurement */ +/* profile information (or NULL) */ +XEN_GUEST_HANDLE_64 (xen_sysctl_lockprof_data_t) data; +}; +typedef struct xen_sysctl_lockprof_op xen_sysctl_lockprof_op_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_op_t); +struct xen_sysctl {uint32_t cmd;uint32_t interface_version;/* XEN_SYSCTL_INTERFACE_VERSION */
@@ -471,6 +503,7 @@struct xen_sysctl_cpu_hotplug cpu_hotplug;struct xen_sysctl_pm_op pm_op;struct xen_sysctl_page_offline_op page_offline; +struct xen_sysctl_lockprof_op lockprof_op;uint8_t pad[128 ]; } u; };
xen/include/xen/sched.h +2 -0
@@ -276,6 +276,8 @@/* transcendent memory, auto-allocated on first tmem op by each domain */ void *tmem; + +SPIN_LOCK_PROF_ARRAY (5 ); };struct domain_setup_info
xen/include/xen/spinlock.h +103 -1
120 lines not on this page · raw message
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 6 lines · Juergen Gross
Quoted 2 lines
Okay, well this all sounds an acceptable kind of direction. Worth spinning another patch.
Here it is.
I managed to avoid the perfc-like stuff completely.
Quoted 6 lines
Included are 2 global locks defined for profiling and 2 locks in the domain structure.
I've added some comments in
spinlock.hhow to use the profiling in structures.Tested by compilation and boot of Dom0 with and without profiling enabled.
Not keen on exposing the lock_profile_funcs[] stuff; that and a few other aspects are ratehr clumsy looking. I would rather dynamically register/unregister lock-profile blocks; something like:
/* domain creation */
lock_profile_register_struct(
LOCKPROF_TYPE_PERDOMAIN, d, d->domid);
/* domain destruction */
lock_profile_unregister_struct(d);
Requires a 'struct lock_profile lock_profile' or similar to be declared within the domain structure. No need to hide that definition within a macro I expect, nor to specify in advance the 'max # locks'.
Where lock_profile is a struct containing the 'idx' (domid in this case) and the head of a linked list of locks, which gets appended to whenever a new pofiled lock is declared. All locks can go back to having the entire profile struct embedded within them - I don't care about the space wastage for unprofiled locks. This means that lock_profile is a fixed-size struct and does not need to contain the profiling data itself -- it just points at a chain of locks that extends on each spin_lock_init_prof().
lock_profile_register_struct() adds the lock_profile_struct to a linked list of such structs for this LOCKPROF_TYPE (so you keep an array of list_heads indexed by LOCKPROF_TYPE, and you can protect that array and all sub-linked-lists with a big rwlock). Something like:
static DEFINE_RWLOCK lock_profiles_lock;
static struct lock_profile *lock_profiles[NR_LOCKPROF_TYPES]
Or somesuch.
Regarding macro style, crap like starting a multi-statement macro with a semi-colon, and not terminating it with one (e.g., _spin_lock_init_prof()) is unacceptable style. The C idiom is to enclose multiple statements within a do{}while(0), or possibly the gcc idiom {()}.
-- Keir
Quoted 5 lines · Juergen Gross
It helped me a lot for narrowing down my performance problem with a 8 vcpu BS2000 system (this was the main reason I made the patch). Finding such a problem in xentrace is quite a bit of work. xentrace produced over 500 MB of data in 30 seconds, while reading the xenlockprof output revealed the bottleneck in seconds.
Minor point, but IIRC you identified the shadow lock as the problem; but the shadow lock covers a lot of code. Did this patch alone tell you what *aspect* of the shadow code was causing the problem? That's what xentrace was designed to do. :-)
I agree, however, that for a lot of uses, xentrace isn't the right tool. It does introduce a lot of memory and disk overhead. Having this kind of lock profiling, and sample-based profiling like oprofile or gprof for Xen is the right tool for cases where a low-overhead aggregate measurement is all that's necessary.
Xentrace excels when aggregates don't give you enough information, and you need to drill down into specific sequences of events, or see how things change over time. It's also useful for situations where it's inconvenient to run a test iteratively as you add aggregate information (for instance, if it's a customer running the test or you're looking at a problem 3-hours into a 6-hour test). Instead of having Xen make new aggregates, you can use the analysis tool to make new aggregates as you need them.
-George
PATCH 1/1 support of lock profiling in Xen
In reply to Keir FraserQuoted 17 lines · Keir Fraser
Quoted 11 lines · Juergen Gross
Quoted 2 lines
Okay, well this all sounds an acceptable kind of direction. Worth spinning another patch.
Here it is.
I managed to avoid the perfc-like stuff completely.Included are 2 global locks defined for profiling and 2 locks in the domain structure.
I've added some comments in
spinlock.hhow to use the profiling in structures.Tested by compilation and boot of Dom0 with and without profiling enabled.
Not keen on exposing the lock_profile_funcs[] stuff; that and a few other aspects are ratehr clumsy looking. I would rather dynamically register/unregister lock-profile blocks; something like:
Quoted 8 lines
/* domain creation */ lock_profile_register_struct( LOCKPROF_TYPE_PERDOMAIN, d, d->domid); /* domain destruction */ lock_profile_unregister_struct(d);Requires a 'struct
lock_profilelock_profile' or similar to be declared within the domain structure. No need to hide that definition within a macro I expect, nor to specify in advance the 'max # locks'.
Round 3 then :-)
I think a rw-lock is not needed, as reading is a very rare event which will be executed in parallel less then once a year. ;-)
Better?
Juergen
Signature · 5 lines
Juergen Gross Principal Developer Operating Systems TSP ES&S SWE OS6 Telephone: +49 (0) 89 636 47950 Fujitsu Technolgy Solutions e-mail: [email protected] Otto-Hahn-Ring 6 Internet: ts.fujitsu.com D-81739 Muenchen Company details: ts.fujitsu.com/imprint.html
tools/libxc/xc_misc.c +24 -0
@@ -121,6 +121,30 @@return rc; } +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data) +{ +int rc; +DECLARE_SYSCTL ; + + sysctl.cmd = XEN_SYSCTL_lockprof_op; + sysctl.u.lockprof_op.cmd = opcode; + sysctl.u.lockprof_op.max_elem = n_elems ? *n_elems :0 ; +set_xen_guest_handle (sysctl.u.lockprof_op.data, data); + + rc =do_sysctl (xc_handle, &sysctl); + +if (n_elems) + *n_elems = sysctl.u.lockprof_op.nr_elem; +if (time) + *time = sysctl.u.lockprof_op.time; + +return rc; +} +int xc_getcpuinfo (int xc_handle,int max_cpus,xc_cpuinfo_t *info,int *nr_cpus) {
tools/libxc/xenctrl.h +8 -0
@@ -700,6 +700,14 @@int *nbr_desc, int *nbr_val); +typedef xen_sysctl_lockprof_data_t xc_lockprof_data_t ; +/* IMPORTANT: The caller is responsible for mlock()'ing the @data array. */ +int xc_lockprof_control (int xc_handle, +uint32_t opcode, +uint32_t *n_elems, +uint64_t *time, +xc_lockprof_data_t *data); +/** * Memory maps a range within one domain to a local address range. Mappings * should be unmapped with munmap and should follow the same rules as mmap
tools/misc/Makefile +3 -3
@@ -10,7 +10,7 @@ HDRS = $(wildcard *.h) -TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat +TARGETS-y := xenperf xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof TARGETS-$(CONFIG_X86) += xen-detect xen-hvmctx TARGETS := $(TARGETS-y)
@@ -22,7 +22,7 @@ INSTALL_BIN-$(CONFIG_X86) += xen-detect INSTALL_BIN := $(INSTALL_BIN-y) -INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat +INSTALL_SBIN-y := xm xen-bugtool xen-python-path xend xenperf xsview xenpm xen-tmem-list-parse gtraceview gtracestat xenlockprof INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx INSTALL_SBIN := $(INSTALL_SBIN-y)
@@ -49,7 +49,7 @@ %.o: %.c $(HDRS) Makefile $(CC) -c $(CFLAGS) -o $@ $< -xen-hvmctx xenperf xenpm gtracestat: %: %.o Makefile +xen-hvmctx xenperf xenpm gtracestat xenlockprof: %: %.o Makefile $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LDFLAGS_libxenctrl) gtraceview: %: %.o Makefile
tools/misc/xenlockprof.c new file +136 -0
136 lines not on this page · raw message
xen/Rules.mk +2 -0
@@ -6,6 +6,7 @@ verbose ?= n perfc ?= n perfc_arrays ?= n +lock_profile ?= n crash_debug ?= n frame_pointer ?= n
@@ -49,6 +50,7 @@ CFLAGS-$(crash_debug) += -DCRASH_DEBUG CFLAGS-$(perfc) += -DPERF_COUNTERS CFLAGS-$(perfc_arrays) += -DPERF_ARRAYS +CFLAGS-$(lock_profile) += -DLOCK_PROFILE CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER ifneq ($(max_phys_cpus),)
xen/arch/x86/domain.c +2 -0
@@ -32,6 +32,7 @@#include <xen/acpi.h> #include <xen/pci.h> #include <xen/paging.h> +#include <public/sysctl.h> #include <asm/regs.h> #include <asm/mc146818rtc.h> #include <asm/system.h>
@@ -185,6 +186,7 @@void free_domain_struct (struct domain *d) { +lock_profile_deregister_struct (LOCKPROF_TYPE_PERDOM , d);free_xenheap_pages (d,get_order_from_bytes (sizeof (*d))); }
xen/arch/x86/xen.lds.S +7 -0
@@ -62,6 +62,13 @@ *(.data.read_mostly) } :text +#ifdef LOCK_PROFILE + . = ALIGN(32); + __lock_profile_start = .; + .lockprofile.data : { *(.lockprofile.data) } :text + __lock_profile_end = .; +#endif + . = ALIGN(4096); /* Init code and data */ __init_begin = .; .init.text : {
xen/common/domain.c +5 -2
@@ -29,6 +29,7 @@#include <acpi/cpufreq/cpufreq.h> #include <asm/debugger.h> #include <public/sched.h> +#include <public/sysctl.h> #include <public/vcpu.h> #include <xsm/xsm.h> #include <xen/trace.h>
@@ -221,13 +222,15 @@memset (d,0 ,sizeof (*d)); d->domain_id = domid; +lock_profile_register_struct (LOCKPROF_TYPE_PERDOM , d, domid,"Domain" ); +if (xsm_alloc_security_domain (d) !=0 ) goto fail; init_status |= INIT_xsm;atomic_set (&d->refcnt,1 ); -spin_lock_init (&d->domain_lock); -spin_lock_init (&d->page_alloc_lock); +spin_lock_init_prof (d, domain_lock); +spin_lock_init_prof (d, page_alloc_lock);spin_lock_init (&d->shutdown_lock);spin_lock_init (&d->hypercall_deadlock_mutex);INIT_PAGE_LIST_HEAD (&d->page_list);
xen/common/keyhandler.c +20 -0
@@ -347,6 +347,20 @@ };#endif +#ifdef LOCK_PROFILE +extern void spinlock_profile_printall (unsigned char key); +static struct keyhandler spinlock_printall_keyhandler = { + .diagnostic =1 , + .u.fn = spinlock_profile_printall, + .desc ="print lock profile info" +}; +extern void spinlock_profile_reset (unsigned char key); +static struct keyhandler spinlock_reset_keyhandler = { + .u.fn = spinlock_profile_reset, + .desc ="reset lock profile info" +}; +#endif +static void run_all_nonirq_keyhandlers (unsigned long unused) {/* Fire all the non-IRQ-context diagnostic keyhandlers */
@@ -428,6 +442,12 @@register_keyhandler ('p' , &perfc_printall_keyhandler);register_keyhandler ('P' , &perfc_reset_keyhandler);#endif + +#ifdef LOCK_PROFILE +register_keyhandler ('l' , &spinlock_printall_keyhandler); +register_keyhandler ('L' , &spinlock_reset_keyhandler); +#endif + }/*
xen/common/spinlock.c +258 -0
337 lines not on this page · raw message
xen/common/sysctl.c +12 -0
@@ -29,6 +29,9 @@extern long arch_do_sysctl (struct xen_sysctl *op,XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl); +#ifdef LOCK_PROFILE +extern int spinlock_profile_control (xen_sysctl_lockprof_op_t *pc); +#endif long do_sysctl (XEN_GUEST_HANDLE (xen_sysctl_t) u_sysctl) {
@@ -144,6 +147,15 @@break ;#endif +#ifdef LOCK_PROFILE +case XEN_SYSCTL_lockprof_op: + { + ret =spinlock_profile_control (&op->u.lockprof_op); +if (copy_to_guest (u_sysctl, op,1 ) ) + ret = -EFAULT ; + } +break ; +#endif case XEN_SYSCTL_debug_keys: {char c;
xen/include/public/sysctl.h +33 -0
@@ -454,6 +454,38 @@#define PG_OFFLINE_OWNER_SHIFT 16 +#define XEN_SYSCTL_lockprof_op 11 +/* Sub-operations: */ +#define XEN_SYSCTL_LOCKPROF_reset 1/* Reset all profile data to zero. */ +#define XEN_SYSCTL_LOCKPROF_query 2/* Get lock profile information. */ +/* Record-type: */ +#define LOCKPROF_TYPE_GLOBAL 0/* global lock, idx meaningless */ +#define LOCKPROF_TYPE_PERDOM 1/* per-domain lock, idx is domid */ +#define LOCKPROF_TYPE_N 2/* number of types */ +struct xen_sysctl_lockprof_data { +char name[40 ];/* lock name (may include up to 2 %d specifiers) */ +int32_t type;/* LOCKPROF_TYPE_??? */ +int32_t idx;/* index (e.g. domain id) */ +uint64_t lock_cnt;/* # of locking succeeded */ +uint64_t block_cnt;/* # of wait for lock */ +uint64_t lock_time;/* nsecs lock held */ +uint64_t block_time;/* nsecs waited for lock */ +}; +typedef struct xen_sysctl_lockprof_data xen_sysctl_lockprof_data_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_data_t); +struct xen_sysctl_lockprof_op { +/* IN variables. */ +uint32_t cmd;/* XEN_SYSCTL_LOCKPROF_??? */ +uint32_t max_elem;/* size of output buffer */ +/* OUT variables (query only). */ +uint32_t nr_elem;/* number of elements available */ +uint64_t time;/* nsecs of profile measurement */ +/* profile information (or NULL) */ +XEN_GUEST_HANDLE_64 (xen_sysctl_lockprof_data_t) data; +}; +typedef struct xen_sysctl_lockprof_op xen_sysctl_lockprof_op_t ; +DEFINE_XEN_GUEST_HANDLE (xen_sysctl_lockprof_op_t); +struct xen_sysctl {uint32_t cmd;uint32_t interface_version;/* XEN_SYSCTL_INTERFACE_VERSION */
@@ -471,6 +503,7 @@struct xen_sysctl_cpu_hotplug cpu_hotplug;struct xen_sysctl_pm_op pm_op;struct xen_sysctl_page_offline_op page_offline; +struct xen_sysctl_lockprof_op lockprof_op;uint8_t pad[128 ]; } u; };
xen/include/xen/sched.h +2 -0
@@ -276,6 +276,8 @@/* transcendent memory, auto-allocated on first tmem op by each domain */ void *tmem; + +struct lock_profile_qhead profile_head; };struct domain_setup_info
xen/include/xen/spinlock.h +95 -2
111 lines not on this page · raw message
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 10 lines · Juergen Gross
Quoted 3 lines
Requires a 'struct
lock_profilelock_profile' or similar to be declared within the domain structure. No need to hide that definition within a macro I expect, nor to specify in advance the 'max # locks'.Round 3 then :-)
I think a rw-lock is not needed, as reading is a very rare event which will be executed in parallel less then once a year. ;-)
Better?
Yeah, this is fine I think.
-- Keir