make all performance counter per-cpu
PATCH 1/1 make all performance counter per-cpu
.. avoiding the need to update them with atomic (locked) ops.
Conversion here isn't complete in the sense that many places still use the old per-CPU accessors (which are now redundant). Since the patch is already rather big, I'd prefer replacing those in a subsequent patch.
While doing this, I also converted x86's multicall macros to no longer require inclusion of asm-offsets.h in the respective C file (on IA64 the use of asm-offsets.h in C sources seems more wide spread, hence there I rather used IA64_ prefixes for the otherwise conflicting performance counter indices).
On x86, a few counter increments get moved a little, to avoid duplicate counting of preempted hypercalls.
Also, a few counters are being added.
IA64 changes only compile-tested, hence somebody doing active IA64 work may want to have a close look at those changes.
Signed-off-by Jan Beulich <[email protected]>
xen/arch/ia64/asm-offsets.c +5 -4
@@ -223,10 +223,11 @@ void foo(void)#ifdef PERF_COUNTERS BLANK (); -DEFINE (RECOVER_TO_PAGE_FAULT_PERFC_OFS , offsetof (struct perfcounter , recover_to_page_fault)); -DEFINE (RECOVER_TO_BREAK_FAULT_PERFC_OFS , offsetof (struct perfcounter , recover_to_break_fault)); -DEFINE (FAST_HYPERPRIVOP_PERFC_OFS , offsetof (struct perfcounter , fast_hyperprivop)); -DEFINE (FAST_REFLECT_PERFC_OFS , offsetof (struct perfcounter , fast_reflect)); +DEFINE (IA64_PERFC_recover_to_page_fault, PERFC_recover_to_page_fault); +DEFINE (IA64_PERFC_recover_to_break_fault, PERFC_recover_to_break_fault); +DEFINE (IA64_PERFC_fast_vhpt_translate, PERFC_fast_vhpt_translate); +DEFINE (IA64_PERFC_fast_hyperprivop, PERFC_fast_hyperprivop); +DEFINE (IA64_PERFC_fast_reflect, PERFC_fast_reflect);#endif BLANK ();
xen/arch/ia64/xen/hyperprivop.S +6 -7
@@ -26,8 +26,7 @@ # define FAST_HYPERPRIVOPS # ifdef PERF_COUNTERS # define FAST_HYPERPRIVOP_CNT -# define FAST_HYPERPRIVOP_PERFC(N) \ - (perfcounters + FAST_HYPERPRIVOP_PERFC_OFS + (4 * N)) +# define FAST_HYPERPRIVOP_PERFC(N) PERFC(fast_hyperprivop + N) # define FAST_REFLECT_CNT # endif
@@ -364,7 +363,7 @@ GLOBAL_ENTRY(fast_tick_reflect) mov rp=r29;; mov cr.itm=r26;; // ensure next tick #ifdef FAST_REFLECT_CNT - movl r20=perfcounters+FAST_REFLECT_PERFC_OFS+((0x3000>>8)*4);; + movl r20=PERFC(fast_reflect + (0x3000>>8));; ld4 r21=[r20];; adds r21=1,r21;; st4 [r20]=r21;;
@@ -597,7 +596,7 @@ END(fast_break_reflect) // r31 == pr ENTRY(fast_reflect) #ifdef FAST_REFLECT_CNT - movl r22=perfcounters+FAST_REFLECT_PERFC_OFS; + movl r22=PERFC(fast_reflect); shr r23=r20,8-2;; add r22=r22,r23;; ld4 r21=[r22];;
@@ -938,7 +937,7 @@ fast_tlb_no_tr_match: (p7) br.cond.spnt.few page_not_present;; #ifdef FAST_REFLECT_CNT - movl r21=perfcounter+FAST_VHPT_TRANSLATE_PERFC_OFS;; + movl r21=PERFC(fast_vhpt_translate);; ld4 r22=[r21];; adds r22=1,r22;; st4 [r21]=r22;;
@@ -968,7 +967,7 @@ END(fast_tlb_miss_reflect) // we get here if fast_insert fails (e.g. due to metaphysical lookup) ENTRY(recover_and_page_fault) #ifdef PERF_COUNTERS - movl r21=perfcounters + RECOVER_TO_PAGE_FAULT_PERFC_OFS;; + movl r21=PERFC(recover_to_page_fault);; ld4 r22=[r21];; adds r22=1,r22;; st4 [r21]=r22;;
@@ -1832,7 +1831,7 @@ END(hyper_ptc_ga) // recovery block for hyper_itc metaphysical memory lookup ENTRY(recover_and_dispatch_break_fault) #ifdef PERF_COUNTERS - movl r21=perfcounters + RECOVER_TO_BREAK_FAULT_PERFC_OFS;; + movl r21=PERFC(recover_to_break_fault);; ld4 r22=[r21];; adds r22=1,r22;; st4 [r21]=r22;;
xen/arch/ia64/xen/privop_stat.c +46 -42
125 lines not on this page · raw message
xen/arch/ia64/xen/vhpt.c +1 -1
@@ -512,7 +512,7 @@ void gather_vhpt_stats(void)for (i =0 ; i <VHPT_NUM_ENTRIES ; i++, v++)if (!(v->ti_tag &INVALID_TI_TAG )) vhpt_valid++; - perfc_seta(vhpt_valid_entries, cpu, vhpt_valid); + per_cpu(perfcounters, cpu)[PERFC_vhpt_valid_entries] = vhpt_valid; } }#endif
xen/arch/x86/mm.c +8 -3
@@ -1969,6 +1969,8 @@ int do_mmuext_op(if (unlikely (!guest_handle_is_null (pdone)) ) (void )copy_from_guest (&done, pdone,1 ); } +else +perfc_incr (calls_to_mmuext_op);if (unlikely (!guest_handle_okay (uops, count)) ) {
@@ -2223,6 +2225,8 @@ int do_mmuext_op(UNLOCK_BIGLOCK (d); +perfc_add (num_mmuext_ops, i); + out:/* Add incremental work we have done to the @done output parameter. */ if (unlikely (!guest_handle_is_null (pdone)) )
@@ -2257,6 +2261,8 @@ int do_mmu_update(if (unlikely (!guest_handle_is_null (pdone)) ) (void )copy_from_guest (&done, pdone,1 ); } +else +perfc_incr (calls_to_mmu_update);if (unlikely (!guest_handle_okay (ureqs, count)) ) {
@@ -2273,9 +2279,6 @@ int do_mmu_update(domain_mmap_cache_init (&mapcache);domain_mmap_cache_init (&sh_mapcache); -perfc_incrc (calls_to_mmu_update); -perfc_addc (num_page_updates, count); -LOCK_BIGLOCK (d);for ( i =0 ; i < count; i++ )
@@ -2438,6 +2441,8 @@ int do_mmu_update(domain_mmap_cache_destroy (&mapcache);domain_mmap_cache_destroy (&sh_mapcache); +perfc_add (num_page_updates, i); + out:/* Add incremental work we have done to the @done output parameter. */ if (unlikely (!guest_handle_is_null (pdone)) )
xen/arch/x86/x86_32/asm-offsets.c +2 -12
@@ -107,21 +107,11 @@ void __dummy__(void)BLANK ();#if PERF_COUNTERS - OFFSET(PERFC_hypercalls, struct perfcounter, hypercalls); - OFFSET(PERFC_exceptions, struct perfcounter, exceptions); +DEFINE (PERFC_hypercalls, PERFC_hypercalls); +DEFINE (PERFC_exceptions, PERFC_exceptions);BLANK ();#endif - OFFSET(MULTICALL_op, struct multicall_entry, op); - OFFSET(MULTICALL_arg0, struct multicall_entry, args[0]); - OFFSET(MULTICALL_arg1, struct multicall_entry, args[1]); - OFFSET(MULTICALL_arg2, struct multicall_entry, args[2]); - OFFSET(MULTICALL_arg3, struct multicall_entry, args[3]); - OFFSET(MULTICALL_arg4, struct multicall_entry, args[4]); - OFFSET(MULTICALL_arg5, struct multicall_entry, args[5]); - OFFSET(MULTICALL_result, struct multicall_entry, result); -BLANK (); -DEFINE (FIXMAP_apic_base,fix_to_virt (FIX_APIC_BASE ));BLANK ();
xen/arch/x86/x86_32/entry.S +2 -2
@@ -173,7 +173,7 @@ ENTRY(hypercall) GET_CURRENT(%ebx) cmpl $NR_hypercalls,%eax jae bad_hypercall - PERFC_INCR(PERFC_hypercalls, %eax) + PERFC_INCR(PERFC_hypercalls, %eax, %ebx) #ifndef NDEBUG /* Create shadow parameters and corrupt those not used by this call. */ pushl %eax
@@ -429,7 +429,7 @@ handle_exception: movl %esp,%edx pushl %edx # push the cpu_user_regs pointer GET_CURRENT(%ebx) - PERFC_INCR(PERFC_exceptions, %eax) + PERFC_INCR(PERFC_exceptions, %eax, %ebx) call *exception_table(,%eax,4) addl $4,%esp movl UREGS_eflags(%esp),%eax
xen/arch/x86/x86_64/asm-offsets.c +2 -24
@@ -121,30 +121,8 @@ void __dummy__(void)BLANK ();#if PERF_COUNTERS - OFFSET(PERFC_hypercalls, struct perfcounter, hypercalls); - OFFSET(PERFC_exceptions, struct perfcounter, exceptions); -BLANK (); -#endif - - OFFSET(MULTICALL_op, struct multicall_entry, op); - OFFSET(MULTICALL_arg0, struct multicall_entry, args[0]); - OFFSET(MULTICALL_arg1, struct multicall_entry, args[1]); - OFFSET(MULTICALL_arg2, struct multicall_entry, args[2]); - OFFSET(MULTICALL_arg3, struct multicall_entry, args[3]); - OFFSET(MULTICALL_arg4, struct multicall_entry, args[4]); - OFFSET(MULTICALL_arg5, struct multicall_entry, args[5]); - OFFSET(MULTICALL_result, struct multicall_entry, result); -BLANK (); - -#ifdef CONFIG_COMPAT - OFFSET(COMPAT_MULTICALL_op, struct compat_multicall_entry, op); - OFFSET(COMPAT_MULTICALL_arg0, struct compat_multicall_entry, args[0]); - OFFSET(COMPAT_MULTICALL_arg1, struct compat_multicall_entry, args[1]); - OFFSET(COMPAT_MULTICALL_arg2, struct compat_multicall_entry, args[2]); - OFFSET(COMPAT_MULTICALL_arg3, struct compat_multicall_entry, args[3]); - OFFSET(COMPAT_MULTICALL_arg4, struct compat_multicall_entry, args[4]); - OFFSET(COMPAT_MULTICALL_arg5, struct compat_multicall_entry, args[5]); - OFFSET(COMPAT_MULTICALL_result, struct compat_multicall_entry, result); +DEFINE (PERFC_hypercalls, PERFC_hypercalls); +DEFINE (PERFC_exceptions, PERFC_exceptions);BLANK ();#endif
xen/arch/x86/x86_64/compat/entry.S +1 -1
@@ -57,7 +57,7 @@ ENTRY(compat_hypercall) movl UREGS_rbx(%rsp),%edi /* Arg 1 */ #endif leaq compat_hypercall_table(%rip),%r10 - PERFC_INCR(PERFC_hypercalls, %rax) + PERFC_INCR(PERFC_hypercalls, %rax, %rbx) callq *(%r10,%rax,8) #ifndef NDEBUG /* Deliberately corrupt parameter regs used by this hypercall. */
xen/arch/x86/x86_64/entry.S +2 -2
@@ -147,7 +147,7 @@ ENTRY(syscall_enter) pushq UREGS_rip+8(%rsp) #endif leaq hypercall_table(%rip),%r10 - PERFC_INCR(PERFC_hypercalls, %rax) + PERFC_INCR(PERFC_hypercalls, %rax, %rbx) callq *(%r10,%rax,8) #ifndef NDEBUG /* Deliberately corrupt parameter regs used by this hypercall. */
@@ -396,7 +396,7 @@ ENTRY(handle_exception) movl UREGS_entry_vector(%rsp),%eax leaq exception_table(%rip),%rdx GET_CURRENT(%rbx) - PERFC_INCR(PERFC_exceptions, %rax) + PERFC_INCR(PERFC_exceptions, %rax, %rbx) callq *(%rdx,%rax,8) testb $3,UREGS_cs(%rsp) jz restore_all_xen
xen/common/multicall.c +5 -0
@@ -10,6 +10,7 @@#include <xen/event.h> #include <xen/multicall.h> #include <xen/guest_access.h> +#include <xen/perfc.h> #include <asm/current.h> #include <asm/hardirq.h>
@@ -69,14 +70,18 @@ do_multicall(guest_handle_add_offset (call_list,1 ); } +perfc_incr (calls_to_multicall); +perfc_add (calls_from_multicall, nr_calls); mcs->flags =0 ;return 0 ; fault: +perfc_incr (calls_to_multicall); mcs->flags =0 ;return -EFAULT ; preempted: +perfc_add (calls_from_multicall, i); mcs->flags =0 ;return hypercall_create_continuation ( __HYPERVISOR_multicall,"hi" , call_list, nr_calls-i);
xen/common/perfc.c +90 -79
272 lines not on this page · raw message
xen/include/asm-ia64/linux-xen/asm/asmmacro.h +4 -0
@@ -116,4 +116,8 @@ name:# define dv_serialize_instruction#endif +#ifdef PERF_COUNTERS +#define PERFC (n) (THIS_CPU(perfcounters) + (IA64_PERFC_ ## n) * 4) +#endif +#endif /* _ASM_IA64_ASMMACRO_H */
xen/include/asm-ia64/perfc_defn.h +3 -1
@@ -84,7 +84,7 @@ PERFCOUNTER_ARRAY(slow_reflect, "s PERFCOUNTER_ARRAY(fast_reflect, "fast reflection", 0x80) PERFSTATUS(vhpt_nbr_entries, "nbr of entries per VHPT") -PERFSTATUS_CPU(vhpt_valid_entries, "nbr of valid entries in VHPT") +PERFSTATUS(vhpt_valid_entries, "nbr of valid entries in VHPT") PERFCOUNTER_ARRAY(vmx_mmio_access, "vmx_mmio_access", 8)PERFCOUNTER_CPU (vmx_pal_emul,"vmx_pal_emul" )
@@ -106,6 +106,8 @@ PERFSTATUS(privop_addr_##name##_overflow PERFPRIVOPADDR(get_ifa) PERFPRIVOPADDR(thash) + +#undef PERFPRIVOPADDR#endif // vhpt.c
xen/include/asm-ia64/privop_stat.h +9 -16
@@ -1,5 +1,5 @@ -#ifndef _XEN_UA64_PRIVOP_STAT_H -#define _XEN_UA64_PRIVOP_STAT_H +#ifndef _XEN_IA64_PRIVOP_STAT_H +#define _XEN_IA64_PRIVOP_STAT_H#include <asm/config.h> #include <xen/types.h> #include <public/xen.h>
@@ -9,31 +9,24 @@extern void gather_privop_addrs (void );extern void reset_privop_addrs (void ); -#undef PERFCOUNTER#define PERFCOUNTER (var, name) - -#undef PERFCOUNTER_CPU -#define PERFCOUNTER_CPU (var, name) - -#undef PERFCOUNTER_ARRAY#define PERFCOUNTER_ARRAY (var, name, size) -#undef PERFSTATUS#define PERFSTATUS (var, name) - -#undef PERFSTATUS_CPU -#define PERFSTATUS_CPU (var, name) - -#undef PERFSTATUS_ARRAY#define PERFSTATUS_ARRAY (var, name, size) -#undef PERFPRIVOPADDR#define PERFPRIVOPADDR (name) privop_inst_##name,enum privop_inst {#include <asm/perfc_defn.h> }; +#undef PERFCOUNTER +#undef PERFCOUNTER_ARRAY + +#undef PERFSTATUS +#undef PERFSTATUS_ARRAY +#undef PERFPRIVOPADDR#define PRIVOP_COUNT_ADDR (regs,inst) privop_count_addr(regs->cr_iip,inst)
@@ -45,4 +38,4 @@ extern void privop_count_addr(unsigned l#define reset_privop_addrs () do {} while (0)#endif -#endif /* _XEN_UA64_PRIVOP_STAT_H */ +#endif /* _XEN_IA64_PRIVOP_STAT_H */
xen/include/asm-x86/multicall.h +57 -47
141 lines not on this page · raw message
xen/include/asm-x86/perfc_defn.h +5 -3
@@ -18,9 +18,11 @@ PERFCOUNTER_CPU(apic_timer, PERFCOUNTER_CPU(domain_page_tlb_flush, "domain page tlb flushes") -PERFCOUNTER_CPU(calls_to_mmu_update, "calls_to_mmu_update") -PERFCOUNTER_CPU(num_page_updates, "num_page_updates") -PERFCOUNTER_CPU(calls_to_update_va, "calls_to_update_va_map") +PERFCOUNTER(calls_to_mmuext_op, "calls to mmuext_op") +PERFCOUNTER(num_mmuext_ops, "mmuext ops") +PERFCOUNTER(calls_to_mmu_update, "calls to mmu_update") +PERFCOUNTER(num_page_updates, "page updates") +PERFCOUNTER(calls_to_update_va, "calls to update_va_map") PERFCOUNTER_CPU(page_faults, "page faults")PERFCOUNTER_CPU (copy_user_faults,"copy_user faults" )
xen/include/asm-x86/x86_32/asm_defns.h +9 -3
@@ -1,6 +1,8 @@#ifndef __X86_32_ASM_DEFNS_H__#define __X86_32_ASM_DEFNS_H__ +#include <asm/percpu.h> +#ifndef NDEBUG /* Indicate special exception stack frame by inverting the frame pointer. */ #define SETUP_EXCEPTION_FRAME_POINTER \
@@ -47,10 +49,14 @@ 1:#ifdef PERF_COUNTERS -#define PERFC_INCR (_name,_idx) \ - lock incl perfcounters+_name(,_idx,4) +#define PERFC_INCR (_name,_idx,_cur) \ + pushl _cur; \ + movl VCPU_processor(_cur),_cur; \ + shll $PERCPU_SHIFT,_cur; \ + incl per_cpu__perfcounters+_name*4(_cur,_idx,4);\ + popl _cur#else -#define PERFC_INCR (_name,_idx) +#define PERFC_INCR (_name,_idx,_cur)#endif #ifdef CONFIG_X86_SUPERVISOR_MODE_KERNEL
xen/include/asm-x86/x86_64/asm_defns.h +13 -6
@@ -1,6 +1,8 @@#ifndef __X86_64_ASM_DEFNS_H__#define __X86_64_ASM_DEFNS_H__ +#include <asm/percpu.h> +#ifndef NDEBUG /* Indicate special exception stack frame by inverting the frame pointer. */ #define SETUP_EXCEPTION_FRAME_POINTER \
@@ -47,13 +49,18 @@ popq %rdi;#ifdef PERF_COUNTERS -#define PERFC_INCR (_name,_idx) \ - pushq %rdx; \ - leaq perfcounters+_name(%rip),%rdx; \ - lock incl (%rdx,_idx,4); \ - popq %rdx; +#define PERFC_INCR (_name,_idx,_cur) \ + pushq _cur; \ + movslq VCPU_processor(_cur),_cur; \ + pushq %rdx; \ + leaq per_cpu__perfcounters(%rip),%rdx; \ + shlq $PERCPU_SHIFT,_cur; \ + addq %rdx,_cur; \ + popq %rdx; \ + incl _name*4(_cur,_idx,4); \ + popq _cur#else -#define PERFC_INCR (_name,_idx) +#define PERFC_INCR (_name,_idx,_cur)#endif /* Work around AMD erratum #88 */
xen/include/xen/perfc.h +55 -65
179 lines not on this page · raw message
xen/include/xen/perfc_defn.h +3 -0
@@ -6,6 +6,9 @@ PERFCOUNTER_ARRAY(hypercalls, "hypercalls", NR_hypercalls) +PERFCOUNTER(calls_to_multicall, "calls to multicall") +PERFCOUNTER(calls_from_multicall, "calls from multicall") + PERFCOUNTER_CPU(irqs, "#interrupts")PERFCOUNTER_CPU (ipis,"#IPIs" )