X86_64 Ctx switch times - 32bit vs 64bit
I was experimenting with the attached program (taken from an IBM Developerworks article) to find the context switch times on AMD64 machine.
With a 64bit binary I get average 5 to 8 usec/cswitch, whereas the same program compiled as 32bit consistently gives >= 10 usec/cswitch - sometimes even 13 usec/cswitch.
Are there more context switching overheads when running 32bit programs on a 64bit kernel?
Kernel version is 2.6.11-gentoo x86_64.
64bit compile - g++ -O2 -pthread csfast5.cpp -ocsfast64 32bit compile - g++ -m32 -O2 -pthread csfast5.cpp -ocsfast32 Run - ./csfast{32/64} -t 40 -c4 10
Parag
#ifdef _WIN32
# include <windows.h> # define errno GetLastError() # define SLEEP(n) Sleep(1000*(n)) # define CRITS CRITICAL_SECTION # define LOCK EnterCriticalSection # define UNLOCK LeaveCriticalSection # define _WIN32_WINNT 0x0500 # define SLASHC '\\' # define SLASHSTR "\\"
char *facility = "CRITSECT";
HANDLE *th_handles;
typedef HANDLE THREAD_T;
void tstart(LARGE_INTEGER *);
void tend(LARGE_INTEGER *);
double tval(LARGE_INTEGER *, LARGE_INTEGER *);#else
# define WINAPI # include <unistd.h> # include <stdlib.h> # include <string.h> # include <errno.h> # include <sys/types.h> # include <sys/wait.h> # include <sys/time.h> # include <fcntl.h> # include <pthread.h> # define SLEEP(n) sleep(n) # define CRITS pthread_mutex_t # define LOCK pthread_mutex_lock # define UNLOCK pthread_mutex_unlock # define SLASHC '/' # define SLASHSTR "/"
char *facility = "mutex_lock";
pthread_t *th_handles;
typedef pthread_t THREAD_T;
void tstart(struct timeval *);
void tend(struct timeval *);
double tval(struct timeval *, struct timeval *);#endif
typedef struct thrdmem {
unsigned long thrdId;
#ifdef _WIN32
LARGE_INTEGER _tstart;
LARGE_INTEGER _tend;
#else
struct timeval _tstart;
struct timeval _tend;
#endif
int threadnum;
unsigned long tcounter;
} thrdmem_t;CRITS *crits;
int ncrits;thrdmem_t *thrdm;
int nthreads = 2;
int showme = 0;
int csv = 0;
int Unlock(int);
int Lock(int);
void *Malloc(size_t);
#include <stdio.h>
#include <ctype.h>
#define equal !strcmp
#define equaln !strncmp
#define MAXCOUNT 100000
//
// csfast [-d] -[t nthreads] [-c ncrits] [maxcount]
//
// This program does pthread_mutexes and Win2k Critical Sections // These are the fastest thread synchronization primitives on the // respective platforms.
//
// We create nthreads execution environments and and ncrits locks // (ncrits > nthreads) and pass a token back and forth
// between them as fast as we can. We count the number and times and // produce a context switches per second number.
//
void USAGE();
int do_threads();size_t atoik(char *s);
unsigned long maxcount = MAXCOUNT;
char *applname;
char applnamebuf[256];
unsigned long thrdId; // Thread ID
int main(int ac, char *av[])
{
int ret = 0;
//strcpy(applnamebuf,av[0]);
if(strrchr(av[0],SLASHC))
strcpy(applnamebuf, strrchr(av[0],SLASHC)+1);
else
strcpy(applnamebuf, av[0]);
#ifdef _WIN32
{
char *q;
if((q=strrchr(applnamebuf, '.')))
if(!equal(q+1,"exe"))
strcat(applnamebuf,".exe");
}
#endif
applname = applnamebuf;
if(ac == 1) {
USAGE();
return 0;
}
while(ac > 1) {
if(equal(av[1],"-debug") || equal(av[1],"-d")) {
ac--;
av++;
showme++;
}
else if(equal(av[1],"-csv")) {
ac--;
av++;
csv = 1;
}
else if(equaln(av[1], "-t",2)) {
if(av[1][2] == 0) {
ac--;
av++;
nthreads = atoik(av[1]);
}
else {
nthreads = atoik(&av[1][2]);
}
//if(nthreads > 1000) nthreads = 1000;
if(nthreads < 2) nthreads = 2;
ac--;
av++;
}
else if(equaln(av[1], "-c",2)) {
if(av[1][2] == 0) {
ac--;
av++;
ncrits = atoik(av[1]);
}
else {
ncrits = atoik(&av[1][2]);
}
ac--;
av++;
}
else if(isdigit(av[1][0])) {
maxcount = atoik(av[1]);
ac--;
av++;
if(maxcount == 0)
maxcount = 1;
}
}
//
// There has to be at least 1 more critical section than threads.
//
if(ncrits <= nthreads)
ncrits = nthreads + 1;
ret = do_threads();
return ret;
}void USAGE()
{
printf("%s [-d [-d [-d]]] [-t nthreads] [-c ncrits] [maximum count]\n",applname);
return;
}unsigned long WINAPI threadrun(void * var)
{
unsigned i;
thrdmem_t *t = (thrdmem_t *)var;
int tnum = t->threadnum;
int k = tnum;
int k1;
int counterA = tnum;
Lock(k);
#ifdef _WIN32
Sleep(100);
#else
sleep(1);#endif
tstart(&t->_tstart);
for(i = 0; i < maxcount; i++) {
k1 = k + 1;
if(k1 >= ncrits)
k1 = 0;
Lock(k1);
Unlock(k);
if(showme) {
if(showme > 1) {
printf("T%d\n",tnum); fflush(stdout);
}
else if (showme > 2) {
printf("T%d: i=%d %d\n", tnum,i,counterA); fflush(stdout);
}
}
counterA += nthreads;
k = k1;
t->tcounter++;
}
Unlock(k);
tend(&t->_tend);
if(showme > 0) {
// Don't let my printf's interfere with the timing of other threads.
SLEEP(2+(nthreads/40));
double tim = tval(&t->_tstart, &t->_tend);
printf("%lu %s/thread Context switches in %7.3f sec ",
maxcount, facility, tim);
printf("%7.3f usec/cswitch",
(tim*1e6)/(maxcount*nthreads));
printf("\n");
fflush(stdout);
}
#ifdef _WIN32
ExitThread(0);
#endif
return 0;
}
int Unlock(int k)
{
UNLOCK((CRITS *)&crits[k]);
return 1;
}
int Lock(int k)
{
LOCK((CRITS *)&crits[k]);
return 1;
}int do_threads()
{
int i;
unsigned mem;
//
// creates ncrits critical sections for use by the threads.
// creates nthreads thread memories
// creates nthreads threads and passes a token back and forth.
//
mem = (ncrits+1) * sizeof(CRITS);
//mem = ((mem + 4095)/4096) * 4096;
crits = (CRITS *) Malloc(mem);
mem = (nthreads+1)*sizeof(thrdmem_t);
//mem = ((mem + 4095)/4096) * 4096;
thrdm = (thrdmem_t *) Malloc(mem);
mem = (nthreads+1)*sizeof(THREAD_T);
//mem = ((mem + 4095)/4096) * 4096;
th_handles = (THREAD_T *) Malloc(mem);
for(i = 0; i < ncrits + 1; i++)
#ifdef _WIN32
InitializeCriticalSection(&crits[i]);
#else
pthread_mutex_init(&crits[i],NULL);#endif
//printf("%d Threads\n",nthreads); fflush(stdout);
for(i = 0; i < nthreads; i++) {
thrdm[i].threadnum = i;
#ifdef _WIN32
//printf("\b\b\b\b%4d",i); fflush(stdout);
//if((th_handles[i] = CreateThread(NULL, 4096, threadrun,
if((th_handles[i] = CreateThread(NULL, 8192, threadrun,
(void *)&thrdm[i], NULL, &thrdId)) == NULL) {
printf("Creation of %d thread failed err=%d\n", i,errno);
fflush(stdout);
return 1;
}
thrdm[i].thrdId = thrdId;
#else
int terr;# define DEC ( void *(*)(void*) )
terr = pthread_create(&th_handles[i], NULL,
DEC threadrun, (void *)&thrdm[i]);
if(terr) {
printf("pthread_create %d failed: err=%d\n", i,terr);
printf("%s", strerror(terr));
fflush(stdout);
return 1;
}
#endif
}
//printf("\n"); fflush(stdout);
for(i = 0; i < nthreads; i++) {
//printf("\b\b\b\b%4d",i); fflush(stdout);
#ifdef _WIN32
if(WaitForSingleObject(th_handles[i],INFINITE) == WAIT_FAILED) {
printf("WaitForSingleObject FAILED: err=%d\n",errno);
#else
if(pthread_join(th_handles[i],NULL)) {
printf("pthread_join FAILED: err=%d\n",errno);
#endif
fflush(stdout);
return 1;
}
}
// Check that all threads actually completed their tasks.
if(thrdm[0].tcounter != maxcount) {
printf("Thread 0 did %lu out of %lu work\n",
thrdm[0].tcounter,maxcount);
fflush(stdout);
return 1;
}
for(i = 1; i < nthreads; i++) {
if(thrdm[i].tcounter != thrdm[0].tcounter) {
printf("Thread %d did %lu out of %lu work\n",
i,thrdm[0].tcounter,maxcount);
fflush(stdout);
return 1;
}
}
#ifdef _WIN32
//printf("All Complete\n"); fflush(stdout);#endif
double sum = 0.0;
double sum2 = 0.0;
double maxv, minv;
double avg = 0.0;
double tim;
maxv = minv = tval(&thrdm[0]._tstart, &thrdm[0]._tend);
for(i = 0; i < nthreads; i++) {
tim = tval(&thrdm[i]._tstart, &thrdm[i]._tend);
sum += tim;
sum2 += (tim*tim);
if(tim < minv)
minv = tim;
if(tim > maxv)
maxv = tim;
}
avg = sum/nthreads;
if(csv) {
printf("\"%s\",%lu,%d,%d,",
facility, maxcount, nthreads, ncrits);
printf("%.6f,%.6f,%.6f",
(avg*1e6)/(maxcount*nthreads),
(minv*1e6)/(maxcount*nthreads),
(maxv*1e6)/(maxcount*nthreads));
fflush(stdout);
}
else {
printf("AVG: %lu %s t=%d c=%d in %7.3f sec ",
maxcount, facility, nthreads, ncrits, avg);
printf("%7.3f usec/cswitch",
(avg*1e6)/(maxcount*nthreads));
fflush(stdout);
}
printf("\n");
return 0;
}#include <ctype.h>
size_t atoik(char *s)
{
size_t ret = 0;
size_t base;
if(*s == '0') {
base = 8;
if(*++s == 'x' || *s == 'X') {
base = 16;
s++;
}
}
else
base = 10;
for(; isxdigit(*s); s++) {
if(base == 16)
if(isalpha(*s))
ret = base*ret + (toupper(*s) - 'A');
else
ret = base*ret + (*s - '0');
else if(isdigit(*s))
ret = base*ret + (*s - '0');
else
break;
}
for(; isalpha(*s); s++) {
switch(toupper(*s)) {
case 'K': ret *= 1024; break;
case 'M': ret *= 1024*1024; break;
default:
return ret;
}
}
return ret;
}#ifdef _WIN32
static LARGE_INTEGER freq; // GLOBAL
static int tfirst = 1;
void tstart(LARGE_INTEGER *t)
{
if(tfirst) {
QueryPerformanceFrequency(&freq);
tfirst = 0;
}
QueryPerformanceCounter(t);
}
void tend(LARGE_INTEGER *t)
{
QueryPerformanceCounter(t);
}double tval(LARGE_INTEGER *t1, LARGE_INTEGER *t2)
{
return ((double)t2->QuadPart -
(double)t1->QuadPart)/((double)freq.QuadPart);
}#else
void tstart(struct timeval *t)
{
gettimeofday(t, NULL);
}
void tend(struct timeval *t)
{
gettimeofday(t,NULL);
}double tval(struct timeval *tv1, struct timeval *tv2)
{
double t1, t2;
t1 = (double)tv1->tv_sec + (double)tv1->tv_usec/(1000*1000);
t2 = (double)tv2->tv_sec + (double)tv2->tv_usec/(1000*1000);
return t2-t1;
}
#endif
void *Malloc(size_t sz)
{
char *p;
if(showme) printf("Malloc(%d)=", sz);
p = (char *)malloc(sz);
if(p == NULL) {
(void)printf("malloc(%d) failed\n",sz);
fflush(stdout);
exit(1);
}
memset(p, '\0', sz);
if(showme) printf("%x\n",(unsigned int)p); if(showme) fflush(stdout);
return (void *)p;
}// typedef struct _RTL_CRITICAL_SECTION {
// PRTL_CRITICAL_SECTION_DEBUG DebugInfo; // LONG LockCount; // LONG RecursionCount; // HANDLE OwningThread; // HANDLE LockSemaphore; // ULONG_PTR SpinCount;
// } RTL_CRITICAL_SECTION, *PRTL_CRITICAL_SECTION;
Quoted 9 lines · Parag Warudkar
I was experimenting with the attached program (taken from an IBM Developerworks article) to find the context switch times on AMD64 machine.
With a 64bit binary I get average 5 to 8 usec/cswitch, whereas the same program compiled as 32bit consistently gives >= 10 usec/cswitch - sometimes even 13 usec/cswitch.
Are there more context switching overheads when running 32bit programs on a 64bit kernel?
Should be nearly the same in theory, no. This means 32bit programs use %gs as thread register which is a bit more costly to switch because the kernel uses it internally too, but the difference should be less than that.
I suspect your program is more testing the locks anyways, perhaps there is some other difference in the glibc. e.g. 32bit glibc compiled for pre 686 CPUs has slower locks.
oprofile might provide more clue where the overhead is.
-Andi
Quoted 8 lines
Quoted 2 lines
Are there more context switching overheads when running 32bit programs on a 64bit kernel?
Should be nearly the same in theory, no. This means 32bit programs use %gs as thread register which is a bit more costly to switch because the kernel uses it internally too, but the difference should be less than that.
Quoted 7 lines
I suspect your program is more testing the locks anyways, perhaps there is some other difference in the glibc. e.g. 32bit glibc compiled for pre 686 CPUs has slower locks.
oprofile might provide more clue where the overhead is.
-Andi
I ran the 32 bit and 64 bit versions in a loop for good amount of time and captured oprofile report for vmlinux after each run. For the 32 bit runs the top 5 are - (Also attached are complete reports after 32 bit and 64 bit runs)
samples % symbol name 20820 11.5424 copy_user_generic_c 12990 7.2015 ia32_syscall 10131 5.6165 gs_change 9053 5.0189 __dequeue_signal 7494 4.1546 find_pid
For the 64 bit run top 5 are -
samples % symbol name 10604 8.1075 system_call 8497 6.4965 __switch_to 8041 6.1479 do_signal 7712 5.8963 schedule 6538 4.9987 __dequeue_signal
I am yet to interpret/analyze the above...
Parag
opreport -l /usr/src/linux/vmlinux
CPU: AMD64 processors, speed 797.944 MHz (estimated)
Counted CPU_CLK_UNHALTED events (Cycles outside of halt state) with a unit mask of 0x00 (No unit mask) count 100000
samples % symbol name 20820 11.5424 copy_user_generic_c 12990 7.2015 ia32_syscall 10131 5.6165 gs_change 9053 5.0189 __dequeue_signal 7494 4.1546 find_pid 7313 4.0542 schedule 5728 3.1755 sys_rt_sigsuspend 5043 2.7958 ia32_setup_sigcontext 4577 2.5374 __group_complete_signal 4308 2.3883 memcpy 4280 2.3728 do_signal 4250 2.3562 save_i387_ia32 4116 2.2819 acpi_processor_idle 3547 1.9664 retint_restore_args 3510 1.9459 __switch_to 3286 1.8217 recalc_task_prio 3247 1.8001 recalc_sigpending_tsk 3096 1.7164 timer_interrupt 3067 1.7003 get_signal_to_deliver 3062 1.6975 kmem_cache_alloc 2930 1.6244 try_to_wake_up 2632 1.4591 restore_i387_ia32 2620 1.4525 finish_task_switch 2424 1.3438 ia32_setup_frame 2138 1.1853 ia32_cstar_target 2044 1.1332 kmem_cache_free 1988 1.1021 enqueue_task 1927 1.0683 ia32_restore_sigcontext 1756 0.9735 sys32_sigreturn 1500 0.8316 sys_kill 1369 0.7590 group_send_sig_info 1366 0.7573 __group_send_sig_info 1365 0.7567 dequeue_signal 1258 0.6974 kill_proc_info 1177 0.6525 recalc_sigpending 1130 0.6265 dequeue_task 1127 0.6248 send_signal 1062 0.5888 clear_page 1059 0.5871 ia32_ptregs_common 983 0.5450 sys32_rt_sigprocmask 831 0.4607 effective_prio 831 0.4607 int_ret_from_sys_call 724 0.4014 signal_wake_up 722 0.4003 sys_rt_sigprocmask 638 0.3537 task_rq_lock 623 0.3454 find_task_by_pid_type 618 0.3426 retint_swapgs 537 0.2977 sched_clock 524 0.2905 ia32_sysret 492 0.2728 __sigqueue_alloc 427 0.2367 search_by_key 391 0.2168 unmap_vmas 389 0.2157 task_curr 382 0.2118 stub32_rt_sigsuspend 371 0.2057 load_gs_index 346 0.1918 copy_page 305 0.1691 sig_ignored 284 0.1574 __d_lookup 280 0.1552 handle_stop_signal 260 0.1441 check_kill_permission 257 0.1425 ia32_do_syscall 245 0.1358 do_select 243 0.1347 memset 237 0.1314 get_sigframe 235 0.1303 find_vma 233 0.1292 copy_to_user 232 0.1286 deactivate_task 231 0.1281 clear_page_range 227 0.1258 memmove 216 0.1197 int_with_check 210 0.1164 page_fault 204 0.1131 unix_poll 202 0.1120 link_path_walk 199 0.1103 unix_ioctl 183 0.1015 ide_inb 176 0.0976 i8042_interrupt 176 0.0976 thread_return 173 0.0959 __down_interruptible 173 0.0959 system_call 170 0.0942 copy_page_range 161 0.0893 apic_timer_interrupt 158 0.0876 IRQ0x00_interrupt 146 0.0809 copy_from_user 146 0.0809 copy_user_generic 144 0.0798 init_fpu 133 0.0737 free_uid 132 0.0732 do_no_page 130 0.0721 buffered_rmqueue 125 0.0693 __mod_page_state 125 0.0693 __pollwait 125 0.0693 ide_outb 125 0.0693 remove_wait_queue 118 0.0654 wake_up_state 114 0.0632 __find_get_block 113 0.0626 create_virtual_node 111 0.0615 iret_label 106 0.0588 add_wait_queue 99 0.0549 sock_create_lite 97 0.0538 stub32_sigreturn 94 0.0521 __up_read 94 0.0521 _devfs_search_dir 92 0.0510 __delay 83 0.0460 cstar_do_call 83 0.0460 kfree 82 0.0455 sysret_check 80 0.0444 get_num_ver 80 0.0444 unix_stream_recvmsg 73 0.0405 sd_part_size 71 0.0394 pipe_poll 69 0.0383 profile_hit 68 0.0377 __kmalloc 66 0.0366 vfs_read 65 0.0360 sys_select 61 0.0338 free_hot_cold_page 61 0.0338 unix_stream_sendmsg 60 0.0333 groups_search 59 0.0327 copy_process 57 0.0316 do_page_fault 54 0.0299 __reiserfs_permission 54 0.0299 rw_verify_area 53 0.0294 __alloc_pages 53 0.0294 __down_read 53 0.0294 handle_mm_fault 50 0.0277 sys32_kill 48 0.0266 strncpy_from_user 47 0.0261 pci_conf1_read 47 0.0261 pte_alloc_map 46 0.0255 __do_IRQ 44 0.0244 fix_nodes 42 0.0233 free_pages_bulk 41 0.0227 direntry_part_size 41 0.0227 ret_from_sys_call 40 0.0222 direntry_create_vi 40 0.0222 vfs_write 39 0.0216 dput 39 0.0216 fget 39 0.0216 ide_outsl 38 0.0211 filemap_nopage 38 0.0211 poll_freewait 37 0.0205 free_page_and_swap_cache 36 0.0200 __rmqueue 36 0.0200 ll_rw_block 35 0.0194 error_exit 35 0.0194 fput 35 0.0194 inotify_inode_queue_event 35 0.0194 put_page 34 0.0188 bad_range 33 0.0183 find_vma_prev 33 0.0183 free_block 33 0.0183 inotify_dentry_parent_queue_event 33 0.0183 reiserfs_prepare_for_journal 33 0.0183 run_timer_softirq 33 0.0183 sys_read 32 0.0177 __down_write 32 0.0177 do_mmap_pgoff 31 0.0172 __wake_up 31 0.0172 do_lookup 31 0.0172 sleep_on_timeout 30 0.0166 cond_resched 30 0.0166 do_sync_read 30 0.0166 normal_poll 30 0.0166 sys_poll 29 0.0161 __wake_up_bit 29 0.0161 bit_waitqueue 29 0.0161 handle_IRQ_event 29 0.0161 path_lookup 29 0.0161 scheduler_tick 28 0.0155 cache_alloc_refill 28 0.0155 do_readv_writev 28 0.0155 ide_insl 28 0.0155 mark_page_accessed 28 0.0155 permission 28 0.0155 update_process_times 27 0.0150 indirect_part_size 27 0.0150 irq_exit 27 0.0150 rb_erase 27 0.0150 sock_aio_read 26 0.0144 __page_cache_release 26 0.0144 copy_mount_options 25 0.0139 do_softirq 25 0.0139 do_wait 25 0.0139 ide_outbsync 25 0.0139 radix_tree_lookup 25 0.0139 schedule_timeout 25 0.0139 sys_mprotect 25 0.0139 vma_adjust 24 0.0133 do_munmap 24 0.0133 do_wp_page 24 0.0133 rb_insert_color 24 0.0133 update_mem_hiwater 24 0.0133 zone_watermark_ok 23 0.0128 pipe_writev 23 0.0128 skb_dequeue 23 0.0128 xattr_lookup_poison 22 0.0122 fget_light 22 0.0122 find_get_page 22 0.0122 raise_softirq 22 0.0122 reiserfs_restore_prepared_buffer 22 0.0122 sk_alloc 22 0.0122 tty_ldisc_try 21 0.0116 __wake_up_common 21 0.0116 direct_part_size 21 0.0116 do_timer 21 0.0116 sys_rt_sigreturn 21 0.0116 tty_poll 21 0.0116 vfs_ioctl 20 0.0111 account_system_time 20 0.0111 do_journal_end 20 0.0111 in_group_p 20 0.0111 journal_mark_dirty 20 0.0111 wait_tb_buffers_until_unlocked 19 0.0105 find_vma_prepare 19 0.0105 number 19 0.0105 unfix_nodes 18 0.0100 __do_softirq 18 0.0100 __kfree_skb 18 0.0100 __mod_timer 18 0.0100 follow_mount 18 0.0100 int_careful 18 0.0100 sock_alloc_send_skb 17 0.0094 __unix_find_socket_byname 17 0.0094 cpu_idle 17 0.0094 reiserfs_new_inode 17 0.0094 sock_def_error_report 17 0.0094 strnlen_user 16 0.0089 __pagevec_lru_add_active 16 0.0089 alloc_skb 16 0.0089 attach_pid 16 0.0089 device_not_available 16 0.0089 do_balance 16 0.0089 remove_vm_struct 16 0.0089 vsnprintf 15 0.0083 check_rlimit 15 0.0083 do_exit 15 0.0083 do_ioctl 15 0.0083 leaf_insert_into_buf 15 0.0083 reiserfs_update_sd_size 15 0.0083 set_page_dirty 14 0.0078 __getblk 14 0.0078 error_sti 14 0.0078 internal_add_timer 14 0.0078 lru_cache_add_active 14 0.0078 math_state_restore 14 0.0078 reiserfs_permission 14 0.0078 skb_copy_bits 14 0.0078 sys_writev 14 0.0078 unix_dgram_recvmsg 13 0.0072 __get_free_pages 13 0.0072 detach_pid 13 0.0072 dnotify_parent 13 0.0072 notifier_call_chain 13 0.0072 sd_create_vi 13 0.0072 search_by_entry_key 13 0.0072 unix_write_space 13 0.0072 vma_merge 12 0.0067 do_generic_mapping_read 12 0.0067 free_hot_page 12 0.0067 free_pages 12 0.0067 page_add_anon_rmap 12 0.0067 pathrelse_and_restore 12 0.0067 profile_tick 12 0.0067 sigprocmask 12 0.0067 smp_local_timer_interrupt 12 0.0067 sys_access 12 0.0067 update_atime 11 0.0061 IRQ0x0c_interrupt 11 0.0061 __down_read_trylock 11 0.0061 arch_get_unmapped_area 11 0.0061 comp_items 11 0.0061 do_con_write 11 0.0061 expand_files 11 0.0061 kprobe_exceptions_notify 11 0.0061 sys_write 10 0.0055 default_wake_function 10 0.0055 dnotify_flush 10 0.0055 find_next_zero_bit 10 0.0055 get_neighbors 10 0.0055 poll_initwait 10 0.0055 prepare_for_delete_or_cut 10 0.0055 ret_from_intr 10 0.0055 smp_apic_timer_interrupt 10 0.0055 wake_up_bit 9 0.0050 IRQ0x10_interrupt 9 0.0050 __fput 9 0.0050 __rb_rotate_left 9 0.0050 as_insert_request 9 0.0050 copy_files 9 0.0050 do_IRQ 9 0.0050 do_sigaction 9 0.0050 reiserfs_prepare_file_region_for_write 9 0.0050 release_pages 9 0.0050 release_task 9 0.0050 vma_prio_tree_add 8 0.0044 ___pskb_trim 8 0.0044 __exit_signal 8 0.0044 __free_pages 8 0.0044 __up_write 8 0.0044 account_it_prof 8 0.0044 account_user_time 8 0.0044 ack_edge_ioapic_irq 8 0.0044 alloc_inode 8 0.0044 can_share_swap_page 8 0.0044 d_alloc 8 0.0044 dentry_open 8 0.0044 dlci_ioctl_set 8 0.0044 do_sync_write 8 0.0044 generic_delete_inode 8 0.0044 generic_file_read 8 0.0044 generic_fillattr 8 0.0044 get_task_mm 8 0.0044 ide_do_request 8 0.0044 leaf_delete_items_entirely 8 0.0044 open_namei 8 0.0044 page_remove_rmap 8 0.0044 prio_tree_insert 8 0.0044 radix_tree_tag_clear 8 0.0044 skb_copy_and_csum_dev 8 0.0044 sock_aio_write 8 0.0044 verify_iovec 8 0.0044 vma_prio_tree_remove 7 0.0039 __anon_vma_link 7 0.0039 __generic_file_aio_read 7 0.0039 __unhash_process 7 0.0039 __user_walk 7 0.0039 can_dirty 7 0.0039 common_interrupt 7 0.0039 compat_sys_nanosleep 7 0.0039 del_timer 7 0.0039 do_journal_begin_r 7 0.0039 file_read_actor 7 0.0039 flush_old_exec 7 0.0039 get_empty_filp 7 0.0039 getname 7 0.0039 ide_intr 7 0.0039 ide_wait_stat 7 0.0039 journal_begin 7 0.0039 kernel_recvmsg 7 0.0039 leaf_copy_items_entirely 7 0.0039 leaf_move_items 7 0.0039 memcpy_toiovec 7 0.0039 mmput 7 0.0039 note_interrupt 7 0.0039 pci_vector_resources 7 0.0039 pipe_readv 7 0.0039 reiserfs_allocate_blocknrs 7 0.0039 reiserfs_in_journal 7 0.0039 reiserfs_insert_item 7 0.0039 sys_ioctl 7 0.0039 tty_hung_up_p 7 0.0039 unlock_buffer 7 0.0039 vfs_readdir 6 0.0033 __clear_user 6 0.0033 __do_page_cache_readahead 6 0.0033 __find_get_block_slow 6 0.0033 __vm_enough_memory 6 0.0033 alloc_pidmap 6 0.0033 anon_vma_prepare 6 0.0033 cache_reap 6 0.0033 con_chars_in_buffer 6 0.0033 current_fs_time 6 0.0033 error_entry 6 0.0033 indirect_create_vi 6 0.0033 input_event 6 0.0033 load_elf32_binary 6 0.0033 locks_remove_flock 6 0.0033 locks_remove_posix 6 0.0033 n_tty_chars_in_buffer 6 0.0033 pty_chars_in_buffer 6 0.0033 reiserfs_cut_from_item 6 0.0033 reiserfs_find_entry 6 0.0033 reiserfs_get_block 6 0.0033 sched_fork 6 0.0033 set_page_refs 6 0.0033 skb_clone 6 0.0033 sock_sendpage 6 0.0033 split_vma 6 0.0033 sys_rt_sigaction 6 0.0033 tty_ldisc_deref 6 0.0033 unix_dgram_sendmsg 6 0.0033 unix_shutdown 5 0.0028 IRQ0x0f_interrupt 5 0.0028 __vm_stat_account 5 0.0028 _make_cpu_key 5 0.0028 add_save_link 5 0.0028 cascade 5 0.0028 check_right 5 0.0028 clear_inode 5 0.0028 con_write_room 5 0.0028 conv_uni_to_pc 5 0.0028 copy_namespace 5 0.0028 copy_semundo 5 0.0028 cp_new_stat 5 0.0028 direct_create_vi 5 0.0028 do_notify_resume 5 0.0028 leaf_cut_from_buffer 5 0.0028 load_elf_binary 5 0.0028 mousedev_notify_readers 5 0.0028 pathrelse 5 0.0028 pci_bus_read_config_word 5 0.0028 r5_hash 5 0.0028 radix_tree_tag_set 5 0.0028 read_chan 5 0.0028 reiserfs_delete_solid_item 5 0.0028 sched_exit 5 0.0028 sd_is_left_mergeable 5 0.0028 search_for_position_by_key 5 0.0028 serio_interrupt 5 0.0028 sock_readv 5 0.0028 sys32_mmap2 5 0.0028 sys_fcntl 5 0.0028 tty_ldisc_ref_wait 5 0.0028 write_chan 4 0.0022 IRQ0x0e_interrupt 4 0.0022 __block_write_full_page 4 0.0022 __vma_link_rb 4 0.0022 activate_page 4 0.0022 anon_vma_unlink 4 0.0022 cache_flusharray 4 0.0022 cap_capable 4 0.0022 cdrom_pc_intr 4 0.0022 copy_siginfo_to_user 4 0.0022 count 4 0.0022 do_notify_parent 4 0.0022 drain_array_locked 4 0.0022 elv_remove_request 4 0.0022 exit_intr 4 0.0022 fb_flashcursor 4 0.0022 file_ra_state_init 4 0.0022 generic_ide_ioctl 4 0.0022 get_unused_fd 4 0.0022 iget5_locked 4 0.0022 inode_update_time 4 0.0022 keyed_hash 4 0.0022 may_open 4 0.0022 path_release 4 0.0022 pipe_ioctl 4 0.0022 prio_tree_replace 4 0.0022 read_cache_page 4 0.0022 reiserfs_add_entry 4 0.0022 reiserfs_async_progress_wait 4 0.0022 reiserfs_commit_page 4 0.0022 reiserfs_file_write 4 0.0022 reiserfs_lookup 4 0.0022 reiserfs_rmdir 4 0.0022 release_console_sem 4 0.0022 schedule_tail 4 0.0022 shrink_dcache_parent 4 0.0022 skb_clone_fraglist 4 0.0022 skb_drop_fraglist 4 0.0022 skb_queue_purge 4 0.0022 snd_ctl_read 4 0.0022 sock_readv_writev 4 0.0022 sock_sendmsg 4 0.0022 strlen 4 0.0022 submit_bh 4 0.0022 sys_getdents64 4 0.0022 vfs_getattr 4 0.0022 vfs_writev 4 0.0022 vma_prio_tree_insert 4 0.0022 wake_up_new_task 4 0.0022 worker_thread 3 0.0017 SHATransform 3 0.0017 __bio_add_page 3 0.0017 __end_that_request_first 3 0.0017 __free_pages_ok 3 0.0017 __ide_end_request 3 0.0017 __make_request 3 0.0017 __mark_inode_dirty 3 0.0017 __remove_shared_vm_struct 3 0.0017 __set_page_dirty_nobuffers 3 0.0017 __tasklet_schedule 3 0.0017 __vma_link 3 0.0017 arch_unmap_area 3 0.0017 as_completed_request 3 0.0017 as_next_request 3 0.0017 bio_alloc 3 0.0017 blkdev_ioctl 3 0.0017 cdrom_end_request 3 0.0017 check_left 3 0.0017 check_ra_success 3 0.0017 console_conditional_schedule 3 0.0017 d_rehash 3 0.0017 dma_map_sg 3 0.0017 do_fork 3 0.0017 do_setitimer 3 0.0017 do_sigaltstack 3 0.0017 eligible_child 3 0.0017 elv_next_request 3 0.0017 fd_install 3 0.0017 flush_commit_list 3 0.0017 get_far_parent 3 0.0017 get_parents 3 0.0017 get_zeroed_page 3 0.0017 i_attrs_to_sd_attrs 3 0.0017 ide_build_dmatable 3 0.0017 ide_insw 3 0.0017 ide_outl 3 0.0017 inode2sd 3 0.0017 inode_has_buffers 3 0.0017 inotify_inode_is_dead 3 0.0017 journal_end 3 0.0017 lookup_create 3 0.0017 make_le_item_head 3 0.0017 memcpy_fromiovec 3 0.0017 mousedev_event 3 0.0017 mousedev_packet 3 0.0017 mousedev_poll 3 0.0017 mousedev_read 3 0.0017 mpage_writepages 3 0.0017 new_inode 3 0.0017 out_of_line_wait_on_bit_lock 3 0.0017 page_cache_readahead 3 0.0017 page_getlink 3 0.0017 page_waitqueue 3 0.0017 profile_munmap 3 0.0017 radix_tree_gang_lookup_tag 3 0.0017 reiserfs_cache_default_acl 3 0.0017 reiserfs_do_truncate 3 0.0017 reiserfs_paste_into_item 3 0.0017 reiserfs_read_locked_inode 3 0.0017 reiserfs_readdir 3 0.0017 reiserfs_unlink 3 0.0017 rh_report_status 3 0.0017 rm_from_queue 3 0.0017 run_local_timers 3 0.0017 sock_def_readable 3 0.0017 sock_kfree_s 3 0.0017 sys_mmap 3 0.0017 sys_open 3 0.0017 sys_setitimer 3 0.0017 task_timeslice 3 0.0017 test_set_page_writeback 3 0.0017 timespec_trunc 3 0.0017 tty_ioctl 3 0.0017 tty_write 3 0.0017 unix_release_sock 3 0.0017 unlock_page 3 0.0017 vfs_create 3 0.0017 vfs_mkdir 3 0.0017 vfs_rename 3 0.0017 vfs_stat 3 0.0017 vma_link 2 0.0011 B_IS_IN_TREE 2 0.0011 __blk_put_request 2 0.0011 __brelse 2 0.0011 __detach_pid 2 0.0011 __filemap_fdatawrite_range 2 0.0011 __put_task_struct 2 0.0011 __rb_rotate_right 2 0.0011 __rcu_process_callbacks 2 0.0011 __sock_create 2 0.0011 _reiserfs_free_block 2 0.0011 add_timer_randomness 2 0.0011 add_to_page_cache 2 0.0011 add_to_page_cache_lru 2 0.0011 as_may_queue 2 0.0011 as_merge 2 0.0011 as_put_request 2 0.0011 as_queue_empty 2 0.0011 autofs4_revalidate 2 0.0011 autofs4_root_ioctl 2 0.0011 balance_dirty_pages_ratelimited 2 0.0011 can_vma_merge_after 2 0.0011 cdrom_queue_packet_command 2 0.0011 cdrom_read_toc 2 0.0011 cdrom_release 2 0.0011 cdrom_start_packet_command 2 0.0011 compat_sys_wait4 2 0.0011 d_instantiate 2 0.0011 devfs_follow_link 2 0.0011 do_execve 2 0.0011 do_gettimeofday 2 0.0011 do_pipe 2 0.0011 do_posix_clock_monotonic_gettime_parts 2 0.0011 exit_aio 2 0.0011 exit_itimers 2 0.0011 exit_mmap 2 0.0011 exit_thread 2 0.0011 filp_open 2 0.0011 flush_sigqueue 2 0.0011 free_pipe_info 2 0.0011 generic_drop_inode 2 0.0011 generic_write_checks 2 0.0011 get_index 2 0.0011 get_request 2 0.0011 get_rfree 2 0.0011 get_super 2 0.0011 get_unmapped_area 2 0.0011 ide_cdrom_prep_fn 2 0.0011 ide_execute_command 2 0.0011 indirect_is_left_mergeable 2 0.0011 init_page_buffers 2 0.0011 invalidate_bdev 2 0.0011 iput 2 0.0011 journal_mark_freed 2 0.0011 kill_fasync 2 0.0011 kref_get 2 0.0011 kthread_should_stop 2 0.0011 leaf_paste_entries 2 0.0011 leaf_paste_in_buffer 2 0.0011 ll_back_merge_fn 2 0.0011 lookup_hash 2 0.0011 lookup_one_len 2 0.0011 make_cpu_key 2 0.0011 mempool_alloc 2 0.0011 mempool_alloc_slab 2 0.0011 mm_release 2 0.0011 n_tty_receive_room 2 0.0011 page_add_file_rmap 2 0.0011 page_follow_link_light 2 0.0011 pipe_read 2 0.0011 prepare_binprm 2 0.0011 prio_tree_remove 2 0.0011 pskb_put 2 0.0011 ptregscall_common 2 0.0011 pty_write_room 2 0.0011 put_files_struct 2 0.0011 reiserfs_choose_packing 2 0.0011 reiserfs_get_unused_objectid 2 0.0011 reiserfs_prepare_write 2 0.0011 reiserfs_update_inode_transaction 2 0.0011 save_i387 2 0.0011 scan_bitmap_block 2 0.0011 send_sigio_to_task 2 0.0011 seq_printf 2 0.0011 set_fs_pwd 2 0.0011 show_stat 2 0.0011 sk_stop_timer 2 0.0011 sock_recvmsg 2 0.0011 sock_release 2 0.0011 sock_wfree 2 0.0011 sockfd_lookup 2 0.0011 strcmp 2 0.0011 stub_rt_sigreturn 2 0.0011 sys_brk 2 0.0011 sys_munmap 2 0.0011 sys_newfstat 2 0.0011 sys_newlstat 2 0.0011 sys_newstat 2 0.0011 unix_dgram_connect 2 0.0011 wake_up_process 1 5.5e-04 SELECT_DRIVE 1 5.5e-04 __add_entropy_words 1 5.5e-04 __d_path 1 5.5e-04 __elv_add_request 1 5.5e-04 __get_user_4 1 5.5e-04 __insert_inode_hash 1 5.5e-04 __map_syscall32 1 5.5e-04 __pagevec_lru_add 1 5.5e-04 __queue_work 1 5.5e-04 __set_page_dirty_buffers 1 5.5e-04 __unix_insert_socket 1 5.5e-04 __wake_up_sync 1 5.5e-04 _get_block_create_0 1 5.5e-04 add_disk_randomness 1 5.5e-04 add_input_randomness 1 5.5e-04 anon_pipe_buf_release 1 5.5e-04 anon_vma_link 1 5.5e-04 as_antic_stop 1 5.5e-04 as_choose_req 1 5.5e-04 as_latter_request 1 5.5e-04 as_merged_request 1 5.5e-04 as_move_to_dispatch 1 5.5e-04 as_remove_queued_request 1 5.5e-04 as_remove_request 1 5.5e-04 as_set_request 1 5.5e-04 ata_input_data 1 5.5e-04 ata_output_data 1 5.5e-04 atapi_input_bytes 1 5.5e-04 atkbd_interrupt 1 5.5e-04 autofs4_dir_rmdir 1 5.5e-04 autofs4_wait 1 5.5e-04 autofs4_wait_release 1 5.5e-04 autoremove_wake_function 1 5.5e-04 balance_internal 1 5.5e-04 bd_acquire 1 5.5e-04 bio_endio 1 5.5e-04 bio_phys_segments 1 5.5e-04 bio_put 1 5.5e-04 blk_get_backing_dev_info 1 5.5e-04 blk_plug_device 1 5.5e-04 blk_queue_bounce 1 5.5e-04 blk_remove_plug 1 5.5e-04 blkdev_open 1 5.5e-04 calc_deleted_bytes_number 1 5.5e-04 cap_bprm_apply_creds 1 5.5e-04 cap_vm_enough_memory 1 5.5e-04 cdev_put 1 5.5e-04 cdrom_check_status 1 5.5e-04 cdrom_decode_status 1 5.5e-04 cdrom_get_media_event 1 5.5e-04 cdrom_ioctl 1 5.5e-04 cdrom_media_changed 1 5.5e-04 cdrom_open 1 5.5e-04 cdrom_saw_media_change 1 5.5e-04 chrdev_open 1 5.5e-04 complete 1 5.5e-04 console_callback 1 5.5e-04 copy_io_context 1 5.5e-04 copy_item_head 1 5.5e-04 copy_strings 1 5.5e-04 copy_strings_kernel 1 5.5e-04 copy_thread 1 5.5e-04 cpufreq_frequency_table_verify 1 5.5e-04 d_delete 1 5.5e-04 d_invalidate 1 5.5e-04 decrement_bcount 1 5.5e-04 decrement_counters_in_path 1 5.5e-04 destroy_context 1 5.5e-04 devfs_d_delete 1 5.5e-04 direntry_check_left 1 5.5e-04 do_balance_mark_leaf_dirty 1 5.5e-04 do_gettimeoffset_tsc 1 5.5e-04 do_group_exit 1 5.5e-04 do_open 1 5.5e-04 do_set_thread_area 1 5.5e-04 do_sysctl 1 5.5e-04 do_writepages 1 5.5e-04 drive_is_ready 1 5.5e-04 end_edge_ioapic_irq 1 5.5e-04 end_page_writeback 1 5.5e-04 error_kernelspace 1 5.5e-04 eventpoll_init_file 1 5.5e-04 exit_mm 1 5.5e-04 exit_sem 1 5.5e-04 expand_stack 1 5.5e-04 extract_entropy 1 5.5e-04 file_move 1 5.5e-04 filemap_fdatawait 1 5.5e-04 filldir64 1 5.5e-04 filp_close 1 5.5e-04 find_inode 1 5.5e-04 find_mergeable_anon_vma 1 5.5e-04 flush_gart 1 5.5e-04 flush_signal_handlers 1 5.5e-04 flush_thread 1 5.5e-04 free_buffer_head 1 5.5e-04 generic_make_request 1 5.5e-04 get_cnode 1 5.5e-04 get_dcookie 1 5.5e-04 get_empty_nodes 1 5.5e-04 get_lfree 1 5.5e-04 get_stack 1 5.5e-04 get_third_component 1 5.5e-04 ia32_setup_arg_pages 1 5.5e-04 ide_dma_intr 1 5.5e-04 ide_do_drive_cmd 1 5.5e-04 ide_do_rw_cdrom 1 5.5e-04 ide_do_rw_disk 1 5.5e-04 ide_init_drive_cmd 1 5.5e-04 idecd_open 1 5.5e-04 idle_cpu 1 5.5e-04 init_new_context 1 5.5e-04 init_tb_struct 1 5.5e-04 inode_change_ok 1 5.5e-04 inode_get_bytes 1 5.5e-04 invalidate_inode_buffers 1 5.5e-04 invalidate_inode_pages 1 5.5e-04 io_schedule_timeout 1 5.5e-04 is_bad_inode 1 5.5e-04 is_leaf_removable 1 5.5e-04 kmem_freepages 1 5.5e-04 kobject_get 1 5.5e-04 kref_put 1 5.5e-04 leaf_copy_boundary_item 1 5.5e-04 leaf_delete_items 1 5.5e-04 load_aout_binary 1 5.5e-04 load_elf_interp 1 5.5e-04 lookup_bdev 1 5.5e-04 lru_add_drain 1 5.5e-04 lru_cache_add 1 5.5e-04 make_empty_dir_item 1 5.5e-04 max_sane_readahead 1 5.5e-04 mempool_free 1 5.5e-04 mod_timer 1 5.5e-04 move_page_tables 1 5.5e-04 mpage_end_io_read 1 5.5e-04 mpage_readpages 1 5.5e-04 n_tty_receive_buf 1 5.5e-04 nanosleep_restart 1 5.5e-04 new_inode_init 1 5.5e-04 next_thread 1 5.5e-04 nf_queue 1 5.5e-04 notify_change 1 5.5e-04 page_put_link 1 5.5e-04 pci_mmap_page_range 1 5.5e-04 pipe_wait 1 5.5e-04 pipe_write 1 5.5e-04 pipe_write_fasync 1 5.5e-04 pipefs_delete_dentry 1 5.5e-04 prepare_to_wait_exclusive 1 5.5e-04 profile_pc 1 5.5e-04 prune_dcache 1 5.5e-04 put_disk 1 5.5e-04 put_filesystem 1 5.5e-04 put_io_context 1 5.5e-04 queue_delayed_work 1 5.5e-04 radix_tree_gang_lookup 1 5.5e-04 radix_tree_insert 1 5.5e-04 rcu_check_callbacks 1 5.5e-04 reiserfs_alloc_inode 1 5.5e-04 reiserfs_can_fit_pages 1 5.5e-04 reiserfs_create 1 5.5e-04 reiserfs_delete_inode 1 5.5e-04 reiserfs_delete_object 1 5.5e-04 reiserfs_discard_prealloc 1 5.5e-04 reiserfs_file_release 1 5.5e-04 reiserfs_free_block 1 5.5e-04 reiserfs_releasepage 1 5.5e-04 reiserfs_set_page_dirty 1 5.5e-04 reiserfs_writepage 1 5.5e-04 release_thread 1 5.5e-04 remove_save_link 1 5.5e-04 retint_check 1 5.5e-04 rff_action 1 5.5e-04 schedule_work 1 5.5e-04 scsi_cmd_ioctl 1 5.5e-04 send_sig_info 1 5.5e-04 set_parameters 1 5.5e-04 set_task_comm 1 5.5e-04 skb_release_data 1 5.5e-04 snd_ctl_poll 1 5.5e-04 sock_aio_dtor 1 5.5e-04 sock_create_kern 1 5.5e-04 sock_ioctl 1 5.5e-04 sock_wake_async 1 5.5e-04 sock_warn_obsolete_bsdism 1 5.5e-04 sock_wmalloc 1 5.5e-04 sprintf 1 5.5e-04 strlcpy 1 5.5e-04 strncmp 1 5.5e-04 sync_blockdev 1 5.5e-04 sync_sb_inodes 1 5.5e-04 sys32_gettimeofday 1 5.5e-04 sys32_mprotect 1 5.5e-04 sys32_rt_sigaction 1 5.5e-04 sys32_set_thread_area 1 5.5e-04 sys_chdir 1 5.5e-04 sys_close 1 5.5e-04 sys_getppid 1 5.5e-04 sys_getpriority 1 5.5e-04 sys_lseek 1 5.5e-04 sys_mkdir 1 5.5e-04 sys_mount 1 5.5e-04 sys_mremap 1 5.5e-04 sys_nanosleep 1 5.5e-04 sys_rmdir 1 5.5e-04 sys_unlink 1 5.5e-04 sysret_careful 1 5.5e-04 test_clear_page_dirty 1 5.5e-04 test_clear_page_writeback 1 5.5e-04 try_to_release_page 1 5.5e-04 tty_read 1 5.5e-04 unix_dgram_disconnected 1 5.5e-04 unix_find_other 1 5.5e-04 unix_stream_connect 1 5.5e-04 unlock_new_inode 1 5.5e-04 vfs_lstat 1 5.5e-04 vfs_rmdir 1 5.5e-04 wait_for_completion 1 5.5e-04 wait_on_page_writeback_range 1 5.5e-04 write_new_vid
samples % symbol name 10604 8.1075 system_call 8497 6.4965 __switch_to 8041 6.1479 do_signal 7712 5.8963 schedule 6538 4.9987 __dequeue_signal 6431 4.9169 find_pid 5128 3.9207 memcpy 5114 3.9100 __group_complete_signal 4382 3.3503 sys_rt_sigsuspend 3423 2.6171 try_to_wake_up 3235 2.4734 sysret_check 3211 2.4550 recalc_sigpending_tsk 3152 2.4099 copy_user_generic_c 3132 2.3946 get_signal_to_deliver 3127 2.3908 acpi_processor_idle 3021 2.3098 recalc_task_prio 2823 2.1584 sys_rt_sigreturn 2602 1.9894 kmem_cache_alloc 2358 1.8028 kmem_cache_free 2243 1.7149 timer_interrupt 2181 1.6675 enqueue_task 2110 1.6132 dequeue_signal 1884 1.4404 kill_proc_info 1870 1.4297 __group_send_sig_info 1624 1.2417 sys_kill 1414 1.0811 group_send_sig_info 1358 1.0383 send_signal 1234 0.9435 sys_rt_sigprocmask 1179 0.9014 dequeue_task 1108 0.8471 ptregscall_common 1095 0.8372 retint_restore_args 1045 0.7990 stub_rt_sigreturn 968 0.7401 recalc_sigpending 910 0.6958 ret_from_sys_call 847 0.6476 effective_prio 819 0.6262 signal_wake_up 811 0.6201 do_sigaltstack 724 0.5535 find_task_by_pid_type 717 0.5482 finish_task_switch 659 0.5038 stub_rt_sigsuspend 509 0.3892 __sigqueue_alloc 484 0.3701 sched_clock 471 0.3601 retint_swapgs 464 0.3548 task_rq_lock 421 0.3219 task_curr 343 0.2622 search_by_key 302 0.2309 copy_to_user 288 0.2202 check_kill_permission 283 0.2164 handle_stop_signal 275 0.2103 clear_page 261 0.1996 sig_ignored 240 0.1835 copy_from_user 224 0.1713 int_ret_from_sys_call 222 0.1697 unmap_vmas 211 0.1613 thread_return 196 0.1499 memmove 189 0.1445 deactivate_task 186 0.1422 __down_interruptible 179 0.1369 get_stack 178 0.1361 IRQ0x00_interrupt 177 0.1353 apic_timer_interrupt 160 0.1223 wake_up_state 147 0.1124 create_virtual_node 129 0.0986 free_uid 120 0.0917 do_select 116 0.0887 clear_page_range 116 0.0887 ide_inb 99 0.0757 find_vma 91 0.0696 memset 90 0.0688 __find_get_block 78 0.0596 get_num_ver 74 0.0566 page_fault 71 0.0543 __up_read 68 0.0520 unix_poll 65 0.0497 ide_outb 65 0.0497 unix_ioctl 64 0.0489 copy_page 63 0.0482 profile_hit 58 0.0443 remove_wait_queue 55 0.0421 __do_IRQ 49 0.0375 buffered_rmqueue 49 0.0375 pipe_poll 46 0.0352 __pollwait 45 0.0344 link_path_walk 45 0.0344 sock_create_lite 44 0.0336 __mod_page_state 44 0.0336 do_no_page 44 0.0336 i8042_interrupt 43 0.0329 do_wait 42 0.0321 __d_lookup 42 0.0321 int_with_check 40 0.0306 add_wait_queue 40 0.0306 direntry_create_vi 38 0.0291 scheduler_tick 38 0.0291 sd_part_size 37 0.0283 pci_conf1_read 34 0.0260 fix_nodes 33 0.0252 __kmalloc 32 0.0245 __down_read 32 0.0245 update_process_times 31 0.0237 irq_exit 29 0.0222 copy_process 29 0.0222 direct_part_size 29 0.0222 do_softirq 29 0.0222 do_timer 29 0.0222 iret_label 29 0.0222 run_timer_softirq 28 0.0214 do_page_fault 28 0.0214 find_vma_prepare 27 0.0206 handle_mm_fault 27 0.0206 sys_mprotect 27 0.0206 sys_poll 26 0.0199 kfree 26 0.0199 vfs_read 25 0.0191 __do_softirq 25 0.0191 cpu_idle 24 0.0183 find_vma_prev 24 0.0183 free_block 24 0.0183 free_pages_bulk 24 0.0183 handle_IRQ_event 23 0.0176 __alloc_pages 23 0.0176 ide_outsl 23 0.0176 raise_softirq 23 0.0176 ret_from_intr 23 0.0176 sd_create_vi 22 0.0168 number 22 0.0168 reiserfs_prepare_for_journal 21 0.0161 copy_page_range 21 0.0161 do_munmap 21 0.0161 poll_freewait 20 0.0153 ll_rw_block 20 0.0153 pipe_writev 20 0.0153 sleep_on_timeout 19 0.0145 do_mmap_pgoff 18 0.0138 direntry_part_size 18 0.0138 free_hot_cold_page 18 0.0138 journal_mark_dirty 18 0.0138 rb_erase 18 0.0138 reiserfs_restore_prepared_buffer 18 0.0138 strncpy_from_user 18 0.0138 sys_select 17 0.0130 bit_waitqueue 16 0.0122 __down_write 16 0.0122 __mod_timer 16 0.0122 normal_poll 16 0.0122 pte_alloc_map 15 0.0115 __wake_up_bit 15 0.0115 account_it_prof 15 0.0115 eligible_child 15 0.0115 error_exit 15 0.0115 filemap_nopage 15 0.0115 ide_insl 15 0.0115 rw_verify_area 15 0.0115 unfix_nodes 15 0.0115 zone_watermark_ok 14 0.0107 common_interrupt 14 0.0107 do_exit 14 0.0107 do_wp_page 14 0.0107 mark_page_accessed 14 0.0107 pipe_readv 14 0.0107 smp_local_timer_interrupt 14 0.0107 vma_merge 14 0.0107 vsnprintf 13 0.0099 __page_cache_release 13 0.0099 fget 13 0.0099 reiserfs_readdir 13 0.0099 tty_poll 12 0.0092 __wake_up_common 12 0.0092 leaf_delete_items_entirely 12 0.0092 pathrelse_and_restore 12 0.0092 profile_tick 12 0.0092 vfs_write 12 0.0092 vma_adjust 11 0.0084 IRQ0x10_interrupt 11 0.0084 __getblk 11 0.0084 __wake_up 11 0.0084 dput 11 0.0084 fput 11 0.0084 free_hot_page 11 0.0084 path_lookup 11 0.0084 strnlen_user 11 0.0084 unix_stream_sendmsg 11 0.0084 wait_tb_buffers_until_unlocked 11 0.0084 wake_up_bit 10 0.0076 bad_range 10 0.0076 cache_reap 10 0.0076 cond_resched 10 0.0076 do_balance 10 0.0076 update_mem_hiwater 9 0.0069 account_system_time 9 0.0069 generic_delete_inode 9 0.0069 ide_outbsync 9 0.0069 lru_cache_add_active 9 0.0069 schedule_timeout 9 0.0069 strlen 9 0.0069 sys_nanosleep 9 0.0069 unlock_buffer 8 0.0061 __get_free_pages 8 0.0061 __pagevec_lru_add_active 8 0.0061 __reiserfs_permission 8 0.0061 __rmqueue 8 0.0061 __up_write 8 0.0061 cache_alloc_refill 8 0.0061 check_left 8 0.0061 do_arch_prctl 8 0.0061 note_interrupt 8 0.0061 notifier_call_chain 8 0.0061 rb_insert_color 8 0.0061 reiserfs_update_sd_size 8 0.0061 sys_read 8 0.0061 tty_ldisc_try 7 0.0054 attach_pid 7 0.0054 check_rlimit 7 0.0054 expand_files 7 0.0054 inode2sd 7 0.0054 permission 7 0.0054 prio_tree_insert 7 0.0054 radix_tree_lookup 7 0.0054 remove_vm_struct 7 0.0054 seq_printf 7 0.0054 set_page_dirty 7 0.0054 sys_ioctl 7 0.0054 unix_stream_recvmsg 7 0.0054 vfs_ioctl 6 0.0046 IRQ0x0f_interrupt 6 0.0046 __fput 6 0.0046 arch_get_unmapped_area 6 0.0046 cdrom_decode_status 6 0.0046 del_timer 6 0.0046 detach_pid 6 0.0046 direct_create_vi 6 0.0046 do_IRQ 6 0.0046 do_journal_end 6 0.0046 do_readv_writev 6 0.0046 do_sync_read 6 0.0046 end_edge_ioapic_irq 6 0.0046 error_sti 6 0.0046 find_get_page 6 0.0046 get_parents 6 0.0046 get_zeroed_page 6 0.0046 inotify_inode_queue_event 6 0.0046 leaf_cut_from_buffer 6 0.0046 mmput 6 0.0046 put_page 6 0.0046 reiserfs_find_entry 6 0.0046 reiserfs_permission 6 0.0046 release_pages 6 0.0046 sigprocmask 6 0.0046 sys_munmap 6 0.0046 sys_write 6 0.0046 update_atime 5 0.0038 __brelse 5 0.0038 __rb_rotate_right 5 0.0038 as_insert_request 5 0.0038 copy_user_generic 5 0.0038 cp_new_stat 5 0.0038 current_fs_time 5 0.0038 default_wake_function 5 0.0038 do_ioctl 5 0.0038 do_notify_parent 5 0.0038 free_page_and_swap_cache 5 0.0038 free_pages 5 0.0038 get_neighbors 5 0.0038 getname 5 0.0038 ide_do_request 5 0.0038 indirect_part_size 5 0.0038 input_event 5 0.0038 page_add_anon_rmap 5 0.0038 reiserfs_delete_solid_item 5 0.0038 search_by_entry_key 5 0.0038 split_vma 5 0.0038 sys_fcntl 5 0.0038 sys_getdents64 4 0.0031 SHATransform 4 0.0031 __exit_signal 4 0.0031 __rb_rotate_left 4 0.0031 __unix_find_socket_byname 4 0.0031 _reiserfs_free_block 4 0.0031 account_user_time 4 0.0031 add_save_link 4 0.0031 anon_vma_prepare 4 0.0031 cap_capable 4 0.0031 check_right 4 0.0031 comp_items 4 0.0031 copy_namespace 4 0.0031 copy_thread 4 0.0031 do_group_exit 4 0.0031 do_journal_begin_r 4 0.0031 find_next_zero_bit 4 0.0031 get_task_mm 4 0.0031 gs_change 4 0.0031 inotify_dentry_parent_queue_event 4 0.0031 internal_add_timer 4 0.0031 kprobe_exceptions_notify 4 0.0031 load_elf_binary 4 0.0031 make_le_item_head 4 0.0031 may_open 4 0.0031 mousedev_packet 4 0.0031 poll_initwait 4 0.0031 profile_pc 4 0.0031 pty_chars_in_buffer 4 0.0031 read_chan 4 0.0031 reiserfs_cut_from_item 4 0.0031 reiserfs_set_page_dirty 4 0.0031 release_task 4 0.0031 run_local_timers 4 0.0031 snd_ctl_read 4 0.0031 sys_getppid 4 0.0031 sys_unlink 4 0.0031 sys_writev 4 0.0031 timespec_trunc 4 0.0031 vma_prio_tree_add 4 0.0031 wake_up_process 3 0.0023 __anon_vma_link 3 0.0023 __down_read_trylock 3 0.0023 __ide_set_handler 3 0.0023 __unhash_process 3 0.0023 __vm_enough_memory 3 0.0023 __vm_stat_account 3 0.0023 __wake_up_sync 3 0.0023 ack_edge_ioapic_irq 3 0.0023 can_share_swap_page 3 0.0023 cap_vm_enough_memory 3 0.0023 cdrom_ioctl 3 0.0023 decrement_bcount 3 0.0023 device_not_available 3 0.0023 dma_map_sg 3 0.0023 do_fork 3 0.0023 do_generic_mapping_read 3 0.0023 do_lookup 3 0.0023 drain_array_locked 3 0.0023 error_entry 3 0.0023 exit_intr 3 0.0023 fget_light 3 0.0023 flush_sigqueue 3 0.0023 generic_drop_inode 3 0.0023 generic_fillattr 3 0.0023 get_super 3 0.0023 get_unused_fd 3 0.0023 indirect_create_vi 3 0.0023 inode_update_time 3 0.0023 journal_begin 3 0.0023 leaf_copy_items_entirely 3 0.0023 leaf_insert_into_buf 3 0.0023 leaf_move_items 3 0.0023 math_state_restore 3 0.0023 n_tty_chars_in_buffer 3 0.0023 open_namei 3 0.0023 out_of_line_wait_on_bit_lock 3 0.0023 pathrelse 3 0.0023 pipe_write 3 0.0023 prio_tree_replace 3 0.0023 radix_tree_tag_set 3 0.0023 reiserfs_delete_object 3 0.0023 reiserfs_insert_item 3 0.0023 reiserfs_new_inode 3 0.0023 sd_is_left_mergeable 3 0.0023 smp_apic_timer_interrupt 3 0.0023 sock_aio_read 3 0.0023 sock_def_error_report 3 0.0023 sock_ioctl 3 0.0023 sys_open 3 0.0023 task_timeslice 3 0.0023 unix_shutdown 3 0.0023 vfs_getattr 3 0.0023 vfs_unlink 3 0.0023 wake_up_new_task 2 0.0015 __detach_pid 2 0.0015 __end_that_request_first 2 0.0015 __free_pages 2 0.0015 __kfree_skb 2 0.0015 __set_page_dirty_buffers 2 0.0015 __vma_link 2 0.0015 _make_cpu_key 2 0.0015 alloc_pidmap 2 0.0015 anon_pipe_buf_map 2 0.0015 as_next_request 2 0.0015 as_queue_empty 2 0.0015 ata_input_data 2 0.0015 blkdev_open 2 0.0015 calc_deleted_bytes_number 2 0.0015 can_vma_merge_after 2 0.0015 cdrom_pc_intr 2 0.0015 cdrom_queue_packet_command 2 0.0015 clear_all_dirty_bits 2 0.0015 console_conditional_schedule 2 0.0015 copy_item_head 2 0.0015 copy_semundo 2 0.0015 direntry_check_right 2 0.0015 direntry_is_left_mergeable 2 0.0015 dlci_ioctl_set 2 0.0015 dnotify_parent 2 0.0015 do_balance_mark_leaf_dirty 2 0.0015 do_setitimer 2 0.0015 do_sigaction 2 0.0015 do_sync_write 2 0.0015 end_level_ioapic_irq 2 0.0015 exit_mm 2 0.0015 fb_flashcursor 2 0.0015 filp_close 2 0.0015 find_get_pages 2 0.0015 flush_signal_handlers 2 0.0015 follow_mount 2 0.0015 get_cnode 2 0.0015 get_disk 2 0.0015 get_empty_filp 2 0.0015 get_rfree 2 0.0015 get_unmapped_area 2 0.0015 groups_search 2 0.0015 ide_do_rw_cdrom 2 0.0015 ide_intr 2 0.0015 in_group_p 2 0.0015 indirect_is_left_mergeable 2 0.0015 kernel_recvmsg 2 0.0015 kill_fasync 2 0.0015 make_cpu_key 2 0.0015 memcpy_toiovec 2 0.0015 mousedev_notify_readers 2 0.0015 n_tty_receive_room 2 0.0015 page_add_file_rmap 2 0.0015 page_remove_rmap 2 0.0015 pci_vector_resources 2 0.0015 prepare_for_delete_or_cut 2 0.0015 proc_pid_unhash 2 0.0015 pskb_put 2 0.0015 r5_hash 2 0.0015 radix_tree_tag_clear 2 0.0015 reiserfs_add_entry 2 0.0015 reiserfs_allocate_blocknrs 2 0.0015 reiserfs_commit_page 2 0.0015 reiserfs_do_truncate 2 0.0015 reiserfs_file_release 2 0.0015 reiserfs_in_journal 2 0.0015 reiserfs_rmdir 2 0.0015 release_thread 2 0.0015 rh_report_status 2 0.0015 sched_exit 2 0.0015 sched_fork 2 0.0015 set_page_refs 2 0.0015 show_stat 2 0.0015 shrink_dcache_parent 2 0.0015 sys_arch_prctl 2 0.0015 sys_chdir 2 0.0015 sys_close 2 0.0015 sys_mmap 2 0.0015 sys_rt_sigaction 2 0.0015 sysret_careful 2 0.0015 tasklet_action 2 0.0015 tty_ldisc_deref 2 0.0015 vfs_fstat 2 0.0015 vfs_lstat 2 0.0015 vfs_mkdir 2 0.0015 vfs_readdir 2 0.0015 vfs_rmdir 2 0.0015 wait_for_completion 2 0.0015 worker_thread 2 0.0015 write_chan 1 7.6e-04 IRQ0x01_interrupt 1 7.6e-04 IRQ0x0c_interrupt 1 7.6e-04 SELECT_DRIVE 1 7.6e-04 ___pskb_trim 1 7.6e-04 __add_entropy_words 1 7.6e-04 __block_prepare_write 1 7.6e-04 __block_write_full_page 1 7.6e-04 __clear_user 1 7.6e-04 __do_page_cache_readahead 1 7.6e-04 __elv_add_request 1 7.6e-04 __freed_request 1 7.6e-04 __generic_file_aio_read 1 7.6e-04 __get_xa_root 1 7.6e-04 __getblk_slow 1 7.6e-04 __kill_fasync 1 7.6e-04 __mark_inode_dirty 1 7.6e-04 __set_page_dirty_nobuffers 1 7.6e-04 __vma_link_rb 1 7.6e-04 add_timer_randomness 1 7.6e-04 anon_pipe_buf_release 1 7.6e-04 arch_unmap_area 1 7.6e-04 as_add_arq_hash 1 7.6e-04 as_add_arq_rb 1 7.6e-04 as_antic_stop 1 7.6e-04 as_merge 1 7.6e-04 atapi_output_bytes 1 7.6e-04 atkbd_event 1 7.6e-04 bdget 1 7.6e-04 bio_endio 1 7.6e-04 blk_get_queue 1 7.6e-04 blkdev_ioctl 1 7.6e-04 block_ioctl 1 7.6e-04 blockable_page_cache_readahead 1 7.6e-04 can_dirty 1 7.6e-04 cascade 1 7.6e-04 cdrom_check_status 1 7.6e-04 cdrom_do_pc_continuation 1 7.6e-04 cdrom_end_request 1 7.6e-04 cdrom_media_changed 1 7.6e-04 cdrom_read_toc 1 7.6e-04 check_disk_change 1 7.6e-04 check_unthrottle 1 7.6e-04 clear_inode 1 7.6e-04 clear_page_dirty_for_io 1 7.6e-04 complete 1 7.6e-04 con_chars_in_buffer 1 7.6e-04 cond_resched_lock 1 7.6e-04 console_callback 1 7.6e-04 copy_files 1 7.6e-04 copy_strings 1 7.6e-04 cursor_timer_handler 1 7.6e-04 d_instantiate 1 7.6e-04 dcache_dir_open 1 7.6e-04 decrement_counters_in_path 1 7.6e-04 delayed_work_timer_fn 1 7.6e-04 destroy_context 1 7.6e-04 dnotify_flush 1 7.6e-04 do_execve 1 7.6e-04 do_gettimeoffset_tsc 1 7.6e-04 do_mremap 1 7.6e-04 do_pipe 1 7.6e-04 do_posix_clock_monotonic_gettime 1 7.6e-04 do_posix_clock_monotonic_gettime_parts 1 7.6e-04 end_bio_bh_io_sync 1 7.6e-04 end_buffer_write_sync 1 7.6e-04 fd_install 1 7.6e-04 file_read_actor 1 7.6e-04 filemap_fdatawrite 1 7.6e-04 filldir64 1 7.6e-04 filp_open 1 7.6e-04 find_first_zero_bit 1 7.6e-04 find_mergeable_anon_vma 1 7.6e-04 finish_wait 1 7.6e-04 flush_old_exec 1 7.6e-04 free_pipe_info 1 7.6e-04 generic_file_mmap 1 7.6e-04 get_empty_nodes 1 7.6e-04 get_far_parent 1 7.6e-04 get_index 1 7.6e-04 get_request 1 7.6e-04 get_third_component 1 7.6e-04 i8042_timer_func 1 7.6e-04 ide_build_sglist 1 7.6e-04 ide_cdrom_check_media_change_real 1 7.6e-04 ide_cdrom_prep_fn 1 7.6e-04 ide_do_drive_cmd 1 7.6e-04 ide_do_rw_disk 1 7.6e-04 ide_end_request 1 7.6e-04 ide_insw 1 7.6e-04 ide_outl 1 7.6e-04 ide_wait_stat 1 7.6e-04 iget5_locked 1 7.6e-04 init_cdrom_command 1 7.6e-04 init_once 1 7.6e-04 init_tb_struct 1 7.6e-04 inode_add_bytes 1 7.6e-04 inode_has_buffers 1 7.6e-04 inotify_inode_is_dead 1 7.6e-04 int_restore_rest 1 7.6e-04 internal_delete_pointers_items 1 7.6e-04 iput 1 7.6e-04 is_leaf_removable 1 7.6e-04 kernel_read 1 7.6e-04 keyed_hash 1 7.6e-04 kill_bdev 1 7.6e-04 kmem_freepages 1 7.6e-04 kthread_should_stop 1 7.6e-04 leaf_copy_boundary_item 1 7.6e-04 leaf_delete_items 1 7.6e-04 load_elf_interp 1 7.6e-04 locks_remove_posix 1 7.6e-04 lookup_mnt 1 7.6e-04 lru_add_drain 1 7.6e-04 make_empty_dir_item 1 7.6e-04 mm_alloc 1 7.6e-04 mm_release 1 7.6e-04 mousedev_poll 1 7.6e-04 mpage_writepages 1 7.6e-04 n_tty_ioctl 1 7.6e-04 n_tty_receive_buf 1 7.6e-04 new_inode 1 7.6e-04 path_release 1 7.6e-04 pipe_read 1 7.6e-04 pipe_wait 1 7.6e-04 prepare_to_copy 1 7.6e-04 prio_tree_remove 1 7.6e-04 pty_write_room 1 7.6e-04 radix_tree_gang_lookup_tag 1 7.6e-04 reiserfs_async_progress_wait 1 7.6e-04 reiserfs_cache_default_acl 1 7.6e-04 reiserfs_discard_all_prealloc 1 7.6e-04 reiserfs_end_ordered_io 1 7.6e-04 reiserfs_file_write 1 7.6e-04 reiserfs_get_block 1 7.6e-04 reiserfs_get_unused_objectid 1 7.6e-04 reiserfs_invalidatepage 1 7.6e-04 reiserfs_lookup 1 7.6e-04 reiserfs_mkdir 1 7.6e-04 reiserfs_paste_into_item 1 7.6e-04 reiserfs_release_objectid 1 7.6e-04 reiserfs_unmap_buffer 1 7.6e-04 reiserfs_update_inode_transaction 1 7.6e-04 remove_save_link 1 7.6e-04 retint_check 1 7.6e-04 scan_bitmap_block 1 7.6e-04 sd_check_left 1 7.6e-04 seq_lseek 1 7.6e-04 serio_interrupt 1 7.6e-04 set_fs_pwd 1 7.6e-04 skb_clone 1 7.6e-04 skb_copy_and_csum_dev 1 7.6e-04 skb_copy_bits 1 7.6e-04 skb_queue_purge 1 7.6e-04 sock_readv 1 7.6e-04 sock_readv_writev 1 7.6e-04 sock_sendmsg 1 7.6e-04 sock_sendpage 1 7.6e-04 sys_brk 1 7.6e-04 sys_exit_group 1 7.6e-04 sys_mkdir 1 7.6e-04 sys_newfstat 1 7.6e-04 sys_newlstat 1 7.6e-04 sys_rmdir 1 7.6e-04 sys_wait4 1 7.6e-04 tty_check_change 1 7.6e-04 tty_ioctl 1 7.6e-04 tty_ldisc_ref_wait 1 7.6e-04 tty_write 1 7.6e-04 unix_dgram_sendmsg 1 7.6e-04 unix_write_space 1 7.6e-04 unlock_page 1 7.6e-04 vfs_rename 1 7.6e-04 vma_link 1 7.6e-04 vma_prio_tree_remove
Quoted 6 lines
samples % symbol name 20820 11.5424 copy_user_generic_c 12990 7.2015 ia32_syscall 10131 5.6165 gs_change 9053 5.0189 __dequeue_signal 7494 4.1546 find_pid
Context switch does not even appear. Probably the i386 glibc does something much slower than 64bit. I would compare straces.
-Andi