Skip to contentSkip to conversation

oltp ~10% regression with 2.6.27-rc5 on stoakley machine

PATCH New Lin Ming 4 participants 19 messages Sep 4, 2008 – Sep 26, 2008

Subsystems (5)
Documentation x86 MIPS Scheduler +1 more subsystems
lore
Share & tools
lore

Lin Ming

author
#0 lore raw

Comparing with 2.6.27-rc4, oltp has ~10% regression with 2.6.27-rc5 on 8-core stoakley machine.

Run oltp with 8 threads 120 seconds, vmstat shows much more idle time, about ~30%

procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------

 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
10  0      0 7822824  42240 123740    0    0   312    47  442 1613  3  2 88  6  0
 9  0      0 7822312  42240 123764    0    0     0    16 26691 232566 56 14 30  0  0
13  0      0 7821940  42240 123764    0    0     0    16 26661 228689 54 14 32  0  0
 8  0      0 7821320  42240 123764    0    0     0    16 31508 263765 61 17 23  0  0
12  0      0 7820948  42240 123764    0    0    16    16 28666 242402 57 15 28  0  0
 9  0      0 7820584  42240 123780    0    0     0    16 27107 230804 56 14 30  0  0
10  0      0 7819964  42240 123796    0    0    16   612 27599 244037 55 16 29  0  0
11  0      0 7819356  42240 123796    0    0     0    64 23540 209713 51 13 36  0  0
10  0      0 7819212  42240 123796    0    0     0    32 25674 224205 54 13 32  0  0
10  0      0 7818716  42240 123796    0    0     0    20 30106 257161 59 16 25  0  0
 7  0      0 7818468  42240 123796    0    0     0    16 28356 241551 57 14 29  0  0
10  0      0 7818096  42240 123796    0    0     0    16 39174 273656 64 16 20  0  0
12  0      0 7817724  42240 123796    0    0     0    20 39688 276936 63 16 20  0  0
11  0      0 7817352  42240 123796    0    0     0    16 42543 285192 66 16 18  0  0
 9  0      0 7817352  42240 123796    0    0     0    16 37083 259830 62 14 24  0  0
 8  0      0 7817104  42240 123796    0    0     0    16 37450 259160 61 15 23  0  0
10  0      0 7816516  42240 123796    0    0     0    64 37425 261870 61 16 23  0  0
11  0      0 7815896  42240 123812    0    0    16    16 41558 279320 66 16 18  0  0
 9  0      0 7815648  42240 123812    0    0     0    16 34017 235741 59 14 28  0  0
10  0      0 7815152  42240 123812    0    0     0    16 35642 248888 60 14 26  0  0
 9  0      0 7814532  42240 123812    0    0     0    16 38517 263220 63 15 22  0  0
 9  0      0 7814160  42240 123812    0    0     0    20 35965 246487 61 14 25  0  0
10  0      0 7814036  42240 123812    0    0     0    16 33852 236313 59 13 28  0  0
11  0      0 7813664  42240 123812    0    0     0    16 34958 244819 59 14 27  0  0
10  0      0 7813416  42240 123812    0    0     0    16 26106 202062 53 10 37  0  0
10  0      0 7812672  42240 123812    0    0     0    16 31174 222714 56 12 32  0  0
 9  0      0 7812300  42240 123812    0    0     0   276 25089 196813 52 11 38  0  0
 9  0      0 7812060  42240 123812    0    0     0    16 31877 228004 57 12 31  0  0

Bisect located below patch,
after reverted this patch the regression disappear.

commit 354879bb977e06695993435745f06a0f6d39ce2b

Author: Peter Zijlstra <[email protected]>
Date:   Mon Aug 25 17:15:34 2008 +0200

sched_clock: fix cpu_clock()

This patch fixes 3 issues:

a) it removes the dependency on jiffies, because jiffies are
incremented
by a single CPU, and the tick is not synchronized between CPUs.
Therefore
relying on it to calculate a window to clip whacky TSC values
doesn't work
as it can drift around.

So instead use [GTOD, GTOD+TICK_NSEC) as the window.

b) __update_sched_clock() did (roughly speaking):

delta = sched_clock() - scd->tick_raw;
clock += delta;

Which gives exponential growth, instead of linear.

c) allows the sched_clock_cpu() value to warp the u64 without
breaking.

the results are more reliable sched_clock() deltas:

-Lin Ming

#1 lore raw
Quoted 64 lines · Lin Ming

Comparing with 2.6.27-rc4, oltp has ~10% regression with 2.6.27-rc5 on 8-core stoakley machine.

Run oltp with 8 threads 120 seconds, vmstat shows much more idle time, about ~30%

procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------

 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
10  0      0 7822824  42240 123740    0    0   312    47  442 1613  3  2 88  6  0
 9  0      0 7822312  42240 123764    0    0     0    16 26691 232566 56 14 30  0  0
13  0      0 7821940  42240 123764    0    0     0    16 26661 228689 54 14 32  0  0
 8  0      0 7821320  42240 123764    0    0     0    16 31508 263765 61 17 23  0  0
12  0      0 7820948  42240 123764    0    0    16    16 28666 242402 57 15 28  0  0
 9  0      0 7820584  42240 123780    0    0     0    16 27107 230804 56 14 30  0  0
10  0      0 7819964  42240 123796    0    0    16   612 27599 244037 55 16 29  0  0
11  0      0 7819356  42240 123796    0    0     0    64 23540 209713 51 13 36  0  0
10  0      0 7819212  42240 123796    0    0     0    32 25674 224205 54 13 32  0  0
10  0      0 7818716  42240 123796    0    0     0    20 30106 257161 59 16 25  0  0
 7  0      0 7818468  42240 123796    0    0     0    16 28356 241551 57 14 29  0  0
10  0      0 7818096  42240 123796    0    0     0    16 39174 273656 64 16 20  0  0
12  0      0 7817724  42240 123796    0    0     0    20 39688 276936 63 16 20  0  0
11  0      0 7817352  42240 123796    0    0     0    16 42543 285192 66 16 18  0  0
 9  0      0 7817352  42240 123796    0    0     0    16 37083 259830 62 14 24  0  0
 8  0      0 7817104  42240 123796    0    0     0    16 37450 259160 61 15 23  0  0
10  0      0 7816516  42240 123796    0    0     0    64 37425 261870 61 16 23  0  0
11  0      0 7815896  42240 123812    0    0    16    16 41558 279320 66 16 18  0  0
 9  0      0 7815648  42240 123812    0    0     0    16 34017 235741 59 14 28  0  0
10  0      0 7815152  42240 123812    0    0     0    16 35642 248888 60 14 26  0  0
 9  0      0 7814532  42240 123812    0    0     0    16 38517 263220 63 15 22  0  0
 9  0      0 7814160  42240 123812    0    0     0    20 35965 246487 61 14 25  0  0
10  0      0 7814036  42240 123812    0    0     0    16 33852 236313 59 13 28  0  0
11  0      0 7813664  42240 123812    0    0     0    16 34958 244819 59 14 27  0  0
10  0      0 7813416  42240 123812    0    0     0    16 26106 202062 53 10 37  0  0
10  0      0 7812672  42240 123812    0    0     0    16 31174 222714 56 12 32  0  0
 9  0      0 7812300  42240 123812    0    0     0   276 25089 196813 52 11 38  0  0
 9  0      0 7812060  42240 123812    0    0     0    16 31877 228004 57 12 31  0  0

Bisect located below patch,
after reverted this patch the regression disappear.

commit 354879bb977e06695993435745f06a0f6d39ce2b

Author: Peter Zijlstra <[email protected]>
Date:   Mon Aug 25 17:15:34 2008 +0200

sched_clock: fix cpu_clock()

This patch fixes 3 issues:

a) it removes the dependency on jiffies, because jiffies are
incremented
by a single CPU, and the tick is not synchronized between CPUs.
Therefore
relying on it to calculate a window to clip whacky TSC values
doesn't work
as it can drift around.

So instead use [GTOD, GTOD+TICK_NSEC) as the window.

b) __update_sched_clock() did (roughly speaking):

delta = sched_clock() - scd->tick_raw;
clock += delta;

Quoted 6 lines

Which gives exponential growth, instead of linear.

c) allows the sched_clock_cpu() value to warp the u64 without
breaking.

the results are more reliable sched_clock() deltas:

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

author
#2 lore raw
Quoted 70 lines · Peter Zijlstra
Quoted 69 lines · Lin Ming

Comparing with 2.6.27-rc4, oltp has ~10% regression with 2.6.27-rc5 on 8-core stoakley machine.

Run oltp with 8 threads 120 seconds, vmstat shows much more idle time, about ~30%

procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------

 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
10  0      0 7822824  42240 123740    0    0   312    47  442 1613  3  2 88  6  0
 9  0      0 7822312  42240 123764    0    0     0    16 26691 232566 56 14 30  0  0
13  0      0 7821940  42240 123764    0    0     0    16 26661 228689 54 14 32  0  0
 8  0      0 7821320  42240 123764    0    0     0    16 31508 263765 61 17 23  0  0
12  0      0 7820948  42240 123764    0    0    16    16 28666 242402 57 15 28  0  0
 9  0      0 7820584  42240 123780    0    0     0    16 27107 230804 56 14 30  0  0
10  0      0 7819964  42240 123796    0    0    16   612 27599 244037 55 16 29  0  0
11  0      0 7819356  42240 123796    0    0     0    64 23540 209713 51 13 36  0  0
10  0      0 7819212  42240 123796    0    0     0    32 25674 224205 54 13 32  0  0
10  0      0 7818716  42240 123796    0    0     0    20 30106 257161 59 16 25  0  0
 7  0      0 7818468  42240 123796    0    0     0    16 28356 241551 57 14 29  0  0
10  0      0 7818096  42240 123796    0    0     0    16 39174 273656 64 16 20  0  0
12  0      0 7817724  42240 123796    0    0     0    20 39688 276936 63 16 20  0  0
11  0      0 7817352  42240 123796    0    0     0    16 42543 285192 66 16 18  0  0
 9  0      0 7817352  42240 123796    0    0     0    16 37083 259830 62 14 24  0  0
 8  0      0 7817104  42240 123796    0    0     0    16 37450 259160 61 15 23  0  0
10  0      0 7816516  42240 123796    0    0     0    64 37425 261870 61 16 23  0  0
11  0      0 7815896  42240 123812    0    0    16    16 41558 279320 66 16 18  0  0
 9  0      0 7815648  42240 123812    0    0     0    16 34017 235741 59 14 28  0  0
10  0      0 7815152  42240 123812    0    0     0    16 35642 248888 60 14 26  0  0
 9  0      0 7814532  42240 123812    0    0     0    16 38517 263220 63 15 22  0  0
 9  0      0 7814160  42240 123812    0    0     0    20 35965 246487 61 14 25  0  0
10  0      0 7814036  42240 123812    0    0     0    16 33852 236313 59 13 28  0  0
11  0      0 7813664  42240 123812    0    0     0    16 34958 244819 59 14 27  0  0
10  0      0 7813416  42240 123812    0    0     0    16 26106 202062 53 10 37  0  0
10  0      0 7812672  42240 123812    0    0     0    16 31174 222714 56 12 32  0  0
 9  0      0 7812300  42240 123812    0    0     0   276 25089 196813 52 11 38  0  0
 9  0      0 7812060  42240 123812    0    0     0    16 31877 228004 57 12 31  0  0

Bisect located below patch,
after reverted this patch the regression disappear.

commit 354879bb977e06695993435745f06a0f6d39ce2b

Author: Peter Zijlstra <[email protected]>
Date:   Mon Aug 25 17:15:34 2008 +0200

sched_clock: fix cpu_clock()

This patch fixes 3 issues:

a) it removes the dependency on jiffies, because jiffies are
incremented
by a single CPU, and the tick is not synchronized between CPUs.
Therefore
relying on it to calculate a window to clip whacky TSC values
doesn't work
as it can drift around.

So instead use [GTOD, GTOD+TICK_NSEC) as the window.

b) __update_sched_clock() did (roughly speaking):

delta = sched_clock() - scd->tick_raw;
clock += delta;

Which gives exponential growth, instead of linear.

c) allows the sched_clock_cpu() value to warp the u64 without
breaking.

Quoted 6 lines
Quoted 1 line

the results are more reliable sched_clock() deltas:

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

task migration failed because of task_hot, the system is un-balanced?

#3 lore raw
Quoted 7 lines · Lin Ming

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

task migration failed because of task_hot, the system is un-balanced?

would be nice to get a representative (==steady state) scheduler trace from the critical portions of that workload. See:

http://people.redhat.com/mingo/sched-devel.git/readme-tracer.txt

Ingo

author
#4 lore raw
Quoted 8 lines · Ingo Molnar
Quoted 6 lines · Lin Ming

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

Quoted 8 lines
Quoted 1 line

task migration failed because of task_hot, the system is un-balanced?

would be nice to get a representative (==steady state) scheduler trace from the critical portions of that workload. See:

http://people.redhat.com/mingo/sched-devel.git/readme-tracer.txt

Ingo

A huge trace result file, see the attachment for more.

# tracer: sched_switch
#
# TASK-PID CPU# TIMESTAMP FUNCTION
# | | | | |

          mysqld-3799  [07]   151.411009:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411015:   3796:120:R   +  3804:120:S
          mysqld-3796  [00]   151.411018:   3796:120:R ==>  3804:120:R
        sysbench-3807  [07]   151.411020:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411020:   3804:120:S ==>  3796:120:R
          mysqld-3799  [07]   151.411028:   3799:120:R   +  3807:120:S
          mysqld-3796  [00]   151.411029:   3796:120:R   +  3804:120:S
          mysqld-3799  [07]   151.411030:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411031:   3796:120:R ==>  3804:120:R
        sysbench-3807  [07]   151.411032:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411049:   3804:120:S ==>  3796:120:R
          mysqld-3799  [07]   151.411052:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411053:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411055:   3796:120:R   +  3804:120:S
        sysbench-3807  [07]   151.411063:   3807:120:S ==>  3799:120:R
          mysqld-3796  [00]   151.411069:   3796:120:S ==>  3804:120:R
          mysqld-3799  [07]   151.411070:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411071:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411073:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411078:   3804:120:S ==>  3798:120:R
          mysqld-3798  [00]   151.411093:   3798:120:R   +  3806:120:S
          mysqld-3799  [07]   151.411093:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411095:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411104:   3807:120:S ==>  3799:120:R
          mysqld-3799  [07]   151.411114:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411115:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411116:   3807:120:S ==>  3799:120:R
          mysqld-3798  [00]   151.411131:   3798:120:S ==>  3806:120:R
          mysqld-3799  [07]   151.411140:   3799:120:R   +  3807:120:S
        sysbench-3806  [00]   151.411149:   3806:120:S ==>  3801:120:R
          mysqld-3799  [07]   151.411149:   3799:120:S ==>  3807:120:R
          mysqld-3801  [00]   151.411153:   3801:120:R   +  3787:120:S
          mysqld-3801  [00]   151.411154:   3801:120:R ==>  3787:120:R
          tracer-3811  [02]   151.411195:   3811:120:R   +  3812:120:R
          tracer-3811  [02]   151.411198:   3811:120:R ==>  3812:120:R
        sysbench-3807  [07]   151.411201:   3807:120:S ==>     0:140:R
          mysqld-3787  [00]   151.411204:      0:140:R   +  3797:120:S
          mysqld-3787  [00]   151.411208:   3787:120:S ==>  3801:120:R
          <idle>-0     [01]   151.411210:      0:140:R ==>  3797:120:R
          mysqld-3801  [00]   151.411217:   3801:120:R   +  3809:120:S
          mysqld-3797  [01]   151.411225:      0:140:R   +  3800:120:S
          <idle>-0     [03]   151.411228:      0:140:R ==>  3800:120:R
          mysqld-3797  [01]   151.411231:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.411257:      0:140:R   +  3802:120:S
          <idle>-0     [05]   151.411259:      0:140:R ==>  3802:120:R
          mysqld-3800  [03]   151.411266:   3800:120:R   +  3808:120:S
          mysqld-3801  [00]   151.411267:   3801:120:S ==>  3809:120:R
          mysqld-3802  [05]   151.411268:      0:140:R   +  3791:120:S
          <idle>-0     [07]   151.411270:      0:140:R ==>  3791:120:R
          mysqld-3797  [01]   151.411271:   3797:120:S ==>  3805:120:R
        sysbench-3809  [00]   151.411271:   3809:120:R   +  3801:120:S
        sysbench-3805  [01]   151.411274:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.411275:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411277:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411277:   3809:120:R   +  3796:120:S
        sysbench-3810  [05]   151.411279:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.411282:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.411283:   3791:120:R ==>  3803:120:R
        sysbench-3803  [07]   151.411285:   3803:120:S ==>  3791:120:R
        sysbench-3805  [01]   151.411289:   3805:120:S ==>  3797:120:R
        sysbench-3809  [00]   151.411292:   3809:120:S ==>  3796:120:R
          mysqld-3797  [01]   151.411292:   3797:120:R   +  3805:120:S
          mysqld-3796  [00]   151.411297:   3796:120:R   +  3798:120:S
          mysqld-3796  [00]   151.411300:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.411308:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411309:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411312:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.411313:   3791:120:R ==>  3803:120:R
        sysbench-3810  [05]   151.411323:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.411326:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.411332:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411333:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411333:   3791:120:R   +  3803:120:S
        sysbench-3810  [05]   151.411334:   3810:120:S ==>  3802:120:R
          mysqld-3796  [00]   151.411341:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411344:   3804:120:R   +  3796:120:S
        sysbench-3804  [00]   151.411356:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.411357:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411358:   3802:120:R ==>  3810:120:R
          mysqld-3796  [00]   151.411359:   3796:120:R   +  3804:120:S
          mysqld-3791  [07]   151.411363:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.411366:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411370:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.411374:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.411377:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.411378:   3791:120:R   +  3803:120:S
          mysqld-3796  [00]   151.411394:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411396:   3804:120:R   +  3796:120:S
          mysqld-3797  [01]   151.411402:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.411405:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.411406:   3802:120:S ==>  3810:120:R
        sysbench-3804  [00]   151.411407:   3804:120:S ==>  3796:120:R
        sysbench-3810  [05]   151.411409:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.411410:   3791:120:S ==>  3803:120:R
          mysqld-3796  [00]   151.411410:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.411412:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411420:   3810:120:S ==>  3802:120:R
        sysbench-3805  [01]   151.411422:   3805:120:S ==>  3797:120:R
          mysqld-3802  [05]   151.411423:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.411424:   3803:120:S ==>  3791:120:R
          mysqld-3797  [01]   151.411426:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.411427:   3791:120:R   +  3803:120:S
          mysqld-3796  [00]   151.411443:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411445:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.411454:   3802:120:S ==>  3810:120:R
        sysbench-3804  [00]   151.411456:   3804:120:S ==>  3796:120:R

# tracer: sched_switch
#
# TASK-PID CPU# TIMESTAMP FUNCTION
# | | | | |

          mysqld-3799  [07]   151.411009:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411015:   3796:120:R   +  3804:120:S
          mysqld-3796  [00]   151.411018:   3796:120:R ==>  3804:120:R
        sysbench-3807  [07]   151.411020:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411020:   3804:120:S ==>  3796:120:R
          mysqld-3799  [07]   151.411028:   3799:120:R   +  3807:120:S
          mysqld-3796  [00]   151.411029:   3796:120:R   +  3804:120:S
          mysqld-3799  [07]   151.411030:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411031:   3796:120:R ==>  3804:120:R
        sysbench-3807  [07]   151.411032:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411049:   3804:120:S ==>  3796:120:R
          mysqld-3799  [07]   151.411052:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411053:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411055:   3796:120:R   +  3804:120:S
        sysbench-3807  [07]   151.411063:   3807:120:S ==>  3799:120:R
          mysqld-3796  [00]   151.411069:   3796:120:S ==>  3804:120:R
          mysqld-3799  [07]   151.411070:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411071:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411073:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411078:   3804:120:S ==>  3798:120:R
          mysqld-3798  [00]   151.411093:   3798:120:R   +  3806:120:S
          mysqld-3799  [07]   151.411093:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411095:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411104:   3807:120:S ==>  3799:120:R
          mysqld-3799  [07]   151.411114:   3799:120:R   +  3807:120:S
          mysqld-3799  [07]   151.411115:   3799:120:R ==>  3807:120:R
        sysbench-3807  [07]   151.411116:   3807:120:S ==>  3799:120:R
          mysqld-3798  [00]   151.411131:   3798:120:S ==>  3806:120:R
          mysqld-3799  [07]   151.411140:   3799:120:R   +  3807:120:S
        sysbench-3806  [00]   151.411149:   3806:120:S ==>  3801:120:R
          mysqld-3799  [07]   151.411149:   3799:120:S ==>  3807:120:R
          mysqld-3801  [00]   151.411153:   3801:120:R   +  3787:120:S
          mysqld-3801  [00]   151.411154:   3801:120:R ==>  3787:120:R
          tracer-3811  [02]   151.411195:   3811:120:R   +  3812:120:R
          tracer-3811  [02]   151.411198:   3811:120:R ==>  3812:120:R
        sysbench-3807  [07]   151.411201:   3807:120:S ==>     0:140:R
          mysqld-3787  [00]   151.411204:      0:140:R   +  3797:120:S
          mysqld-3787  [00]   151.411208:   3787:120:S ==>  3801:120:R
          <idle>-0     [01]   151.411210:      0:140:R ==>  3797:120:R
          mysqld-3801  [00]   151.411217:   3801:120:R   +  3809:120:S
          mysqld-3797  [01]   151.411225:      0:140:R   +  3800:120:S
          <idle>-0     [03]   151.411228:      0:140:R ==>  3800:120:R
          mysqld-3797  [01]   151.411231:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.411257:      0:140:R   +  3802:120:S
          <idle>-0     [05]   151.411259:      0:140:R ==>  3802:120:R
          mysqld-3800  [03]   151.411266:   3800:120:R   +  3808:120:S
          mysqld-3801  [00]   151.411267:   3801:120:S ==>  3809:120:R
          mysqld-3802  [05]   151.411268:      0:140:R   +  3791:120:S
          <idle>-0     [07]   151.411270:      0:140:R ==>  3791:120:R
          mysqld-3797  [01]   151.411271:   3797:120:S ==>  3805:120:R
        sysbench-3809  [00]   151.411271:   3809:120:R   +  3801:120:S
        sysbench-3805  [01]   151.411274:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.411275:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411277:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411277:   3809:120:R   +  3796:120:S
        sysbench-3810  [05]   151.411279:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.411282:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.411283:   3791:120:R ==>  3803:120:R
        sysbench-3803  [07]   151.411285:   3803:120:S ==>  3791:120:R
        sysbench-3805  [01]   151.411289:   3805:120:S ==>  3797:120:R
        sysbench-3809  [00]   151.411292:   3809:120:S ==>  3796:120:R
          mysqld-3797  [01]   151.411292:   3797:120:R   +  3805:120:S
          mysqld-3796  [00]   151.411297:   3796:120:R   +  3798:120:S
          mysqld-3796  [00]   151.411300:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.411308:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411309:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411312:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.411313:   3791:120:R ==>  3803:120:R
        sysbench-3810  [05]   151.411323:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.411326:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.411332:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411333:   3802:120:R ==>  3810:120:R
          mysqld-3791  [07]   151.411333:   3791:120:R   +  3803:120:S
        sysbench-3810  [05]   151.411334:   3810:120:S ==>  3802:120:R
          mysqld-3796  [00]   151.411341:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411344:   3804:120:R   +  3796:120:S
        sysbench-3804  [00]   151.411356:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.411357:   3802:120:R   +  3810:120:S
          mysqld-3802  [05]   151.411358:   3802:120:R ==>  3810:120:R
          mysqld-3796  [00]   151.411359:   3796:120:R   +  3804:120:S
          mysqld-3791  [07]   151.411363:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.411366:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411370:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.411374:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.411377:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.411378:   3791:120:R   +  3803:120:S
          mysqld-3796  [00]   151.411394:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411396:   3804:120:R   +  3796:120:S
          mysqld-3797  [01]   151.411402:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.411405:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.411406:   3802:120:S ==>  3810:120:R
        sysbench-3804  [00]   151.411407:   3804:120:S ==>  3796:120:R
        sysbench-3810  [05]   151.411409:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.411410:   3791:120:S ==>  3803:120:R
          mysqld-3796  [00]   151.411410:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.411412:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411420:   3810:120:S ==>  3802:120:R
        sysbench-3805  [01]   151.411422:   3805:120:S ==>  3797:120:R
          mysqld-3802  [05]   151.411423:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.411424:   3803:120:S ==>  3791:120:R
          mysqld-3797  [01]   151.411426:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.411427:   3791:120:R   +  3803:120:S
          mysqld-3796  [00]   151.411443:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411445:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.411454:   3802:120:S ==>  3810:120:R
        sysbench-3804  [00]   151.411456:   3804:120:S ==>  3796:120:R
        sysbench-3810  [05]   151.411456:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.411457:   3791:120:S ==>  3803:120:R
          mysqld-3796  [00]   151.411460:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.411460:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411467:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.411469:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.411470:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.411472:   3791:120:R   +  3803:120:S
          mysqld-3796  [00]   151.411494:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411497:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.411500:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411504:   3808:120:R   +  3800:120:S
          mysqld-3802  [05]   151.411506:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.411506:   3791:120:S ==>  3803:120:R
        sysbench-3804  [00]   151.411507:   3804:120:S ==>  3796:120:R
        sysbench-3803  [07]   151.411509:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411509:   3810:120:R   +  3802:120:S
          mysqld-3796  [00]   151.411511:   3796:120:R   +  3804:120:S
        sysbench-3808  [03]   151.411516:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.411517:   3803:120:S ==>  3791:120:R
          mysqld-3800  [03]   151.411519:   3800:120:R   +  3808:120:S
          mysqld-3797  [01]   151.411519:   3797:120:S ==>  3805:120:R
        sysbench-3810  [05]   151.411520:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.411521:   3791:120:R   +  3803:120:S
        sysbench-3805  [01]   151.411522:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.411523:   3802:120:R   +  3810:120:S
        sysbench-3805  [01]   151.411532:   3805:120:S ==>  3797:120:R
          mysqld-3800  [03]   151.411533:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411535:   3808:120:R   +  3800:120:S
          mysqld-3797  [01]   151.411536:   3797:120:R   +  3805:120:S
          mysqld-3796  [00]   151.411546:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411548:   3804:120:R   +  3796:120:S
        sysbench-3808  [03]   151.411553:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.411553:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.411554:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.411556:   3800:120:R   +  3808:120:S
        sysbench-3810  [05]   151.411556:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.411556:   3803:120:R   +  3791:120:S
        sysbench-3804  [00]   151.411559:   3804:120:S ==>  3796:120:R
          mysqld-3796  [00]   151.411562:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.411566:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.411567:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.411569:   3800:120:S ==>  3808:120:R
          mysqld-3791  [07]   151.411569:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.411570:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.411571:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.411578:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411580:   3800:120:R   +  3808:120:S
          mysqld-3796  [00]   151.411597:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411599:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.411603:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.411603:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.411606:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.411606:   3803:120:R   +  3791:120:S
        sysbench-3804  [00]   151.411611:   3804:120:S ==>  3796:120:R
          mysqld-3796  [00]   151.411615:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.411615:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.411618:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.411618:   3800:120:S ==>  3808:120:R
          mysqld-3791  [07]   151.411618:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.411620:   3808:120:R   +  3800:120:S
          mysqld-3802  [05]   151.411621:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.411630:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411632:   3800:120:R   +  3808:120:S
          mysqld-3796  [00]   151.411647:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411649:   3804:120:R   +  3796:120:S
          mysqld-3791  [07]   151.411650:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.411651:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.411653:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.411653:   3810:120:R   +  3802:120:S
        sysbench-3804  [00]   151.411661:   3804:120:S ==>  3796:120:R
        sysbench-3803  [07]   151.411662:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.411663:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.411663:   3800:120:S ==>  3808:120:R
          mysqld-3796  [00]   151.411665:   3796:120:R   +  3804:120:S
          mysqld-3791  [07]   151.411665:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.411666:   3808:120:R   +  3800:120:S
          mysqld-3802  [05]   151.411666:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.411674:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411677:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.411699:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.411701:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.411708:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411711:   3808:120:R   +  3800:120:S
        sysbench-3810  [05]   151.411711:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.411715:   3802:120:R   +  3810:120:S
          mysqld-3796  [00]   151.411719:   3796:120:S ==>  3804:120:R
        sysbench-3808  [03]   151.411720:   3808:120:S ==>  3800:120:R
        sysbench-3804  [00]   151.411721:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.411723:   3800:120:R   +  3808:120:S
        sysbench-3804  [00]   151.411731:   3804:120:S ==>  3796:120:R
          mysqld-3796  [00]   151.411734:   3796:120:R   +  3804:120:S
           sleep-3812  [02]   151.411736:   3812:120:S ==>  3585:120:R
            bash-3585  [02]   151.411751:   3585:120:S ==>  3811:120:R
          mysqld-3797  [01]   151.411756:   3797:120:S ==>  3805:120:R
          mysqld-3800  [03]   151.411757:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411759:   3808:120:R   +  3800:120:S
        sysbench-3805  [01]   151.411760:   3805:120:R   +  3797:120:S
        sysbench-3808  [03]   151.411768:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411771:   3800:120:R   +  3808:120:S
          mysqld-3796  [00]   151.411772:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.411775:   3804:120:R   +  3796:120:S
        sysbench-3805  [01]   151.411777:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.411780:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.411781:   3791:120:S ==>  3803:120:R
        sysbench-3804  [00]   151.411783:   3804:120:S ==>  3796:120:R
        sysbench-3803  [07]   151.411784:   3803:120:R   +  3791:120:S
          mysqld-3796  [00]   151.411787:   3796:120:R   +  3804:120:S
          tracer-3811  [02]   151.411791:   3811:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.411796:   3798:120:R   +  3799:120:S
        sysbench-3803  [07]   151.411801:   3803:120:S ==>  3791:120:R
          mysqld-3798  [02]   151.411802:   3798:120:R   +  3806:120:S
          mysqld-3800  [03]   151.411803:   3800:120:S ==>  3808:120:R
          mysqld-3798  [02]   151.411804:   3798:120:R ==>  3806:120:R
          mysqld-3791  [07]   151.411805:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.411805:   3808:120:R   +  3800:120:S
        sysbench-3806  [02]   151.411806:   3806:120:S ==>  3799:120:R
        sysbench-3808  [03]   151.411814:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411818:   3800:120:R   +  3808:120:S
          mysqld-3799  [02]   151.411818:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.411821:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.411822:   3796:120:S ==>  3804:120:R
        sysbench-3807  [02]   151.411824:   3807:120:S ==>  3799:120:R
        sysbench-3804  [00]   151.411824:   3804:120:R   +  3796:120:S
        sysbench-3804  [00]   151.411834:   3804:120:S ==>  3796:120:R
          mysqld-3796  [00]   151.411836:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.411849:   3802:120:S ==>  3810:120:R
          mysqld-3800  [03]   151.411852:   3800:120:S ==>  3808:120:R
        sysbench-3810  [05]   151.411852:   3810:120:R   +  3802:120:S
        sysbench-3808  [03]   151.411854:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.411862:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411865:   3800:120:R   +  3808:120:S
        sysbench-3810  [05]   151.411868:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.411871:   3802:120:R   +  3810:120:S
          mysqld-3799  [02]   151.411875:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.411876:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.411895:   3800:120:S ==>  3808:120:R
        sysbench-3807  [02]   151.411896:   3807:120:S ==>  3798:120:R
        sysbench-3808  [03]   151.411897:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.411906:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.411909:   3800:120:R   +  3808:120:S
          mysqld-3791  [07]   151.411915:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.411918:   3803:120:R   +  3791:120:S
        sysbench-3803  [07]   151.411928:   3803:120:S ==>  3791:120:R
          mysqld-3798  [02]   151.411929:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.411931:   3798:120:R ==>  3806:120:R
          mysqld-3791  [07]   151.411931:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.411940:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411943:   3808:120:R   +  3800:120:S
        sysbench-3806  [02]   151.411944:   3806:120:S ==>  3799:120:R
        sysbench-3808  [03]   151.411954:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.411955:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.411956:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.411957:   3800:120:R   +  3808:120:S
        sysbench-3807  [02]   151.411958:   3807:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.411986:   3802:120:S ==>  3810:120:R
          mysqld-3799  [02]   151.411989:   3799:120:R   +  3807:120:S
        sysbench-3810  [05]   151.411989:   3810:120:R   +  3802:120:S
          mysqld-3799  [02]   151.411991:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.411993:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.411995:   3808:120:R   +  3800:120:S
          mysqld-3796  [00]   151.411997:   3796:120:S ==>  3804:120:R
        sysbench-3810  [05]   151.411999:   3810:120:S ==>  3802:120:R
        sysbench-3807  [02]   151.412001:   3807:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.412002:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.412004:   3808:120:S ==>  3800:120:R
        sysbench-3804  [00]   151.412004:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.412007:   3800:120:R   +  3808:120:S
          mysqld-3798  [02]   151.412010:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412011:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412013:   3806:120:S ==>  3798:120:R
        sysbench-3804  [00]   151.412021:   3804:120:S ==>  3796:120:R
          mysqld-3797  [01]   151.412023:   3797:120:S ==>  3805:120:R
          mysqld-3796  [00]   151.412025:   3796:120:R   +  3804:120:S
        sysbench-3805  [01]   151.412026:   3805:120:R   +  3797:120:S
        sysbench-3805  [01]   151.412036:   3805:120:S ==>  3797:120:R
          mysqld-3800  [03]   151.412039:   3800:120:S ==>  3808:120:R
          mysqld-3797  [01]   151.412039:   3797:120:R   +  3805:120:S
        sysbench-3808  [03]   151.412041:   3808:120:R   +  3800:120:S
          mysqld-3798  [02]   151.412041:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412043:   3798:120:R ==>  3806:120:R
        sysbench-3808  [03]   151.412050:   3808:120:S ==>  3800:120:R
          mysqld-3797  [01]   151.412051:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412053:   3805:120:R   +  3797:120:S
          mysqld-3800  [03]   151.412054:   3800:120:R   +  3808:120:S
        sysbench-3806  [02]   151.412055:   3806:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.412063:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412064:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412066:   3806:120:S ==>  3799:120:R
        sysbench-3805  [01]   151.412071:   3805:120:S ==>  3797:120:R
          mysqld-3799  [02]   151.412072:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.412073:   3799:120:R ==>  3807:120:R
          mysqld-3797  [01]   151.412074:   3797:120:R   +  3805:120:S
        sysbench-3807  [02]   151.412075:   3807:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.412085:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412087:   3805:120:R   +  3797:120:S
        sysbench-3805  [01]   151.412094:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412096:   3797:120:R   +  3805:120:S
          mysqld-3798  [02]   151.412099:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412101:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412111:   3806:120:S ==>  3799:120:R
          mysqld-3796  [00]   151.412132:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.412135:   3804:120:R   +  3796:120:S
          mysqld-3799  [02]   151.412136:   3799:120:R   +  3807:120:S
          mysqld-3797  [01]   151.412137:   3797:120:S ==>  3805:120:R
          mysqld-3799  [02]   151.412139:   3799:120:R ==>  3807:120:R
        sysbench-3805  [01]   151.412140:   3805:120:R   +  3797:120:S
        sysbench-3804  [00]   151.412144:   3804:120:S ==>  3796:120:R
          mysqld-3796  [00]   151.412147:   3796:120:R   +  3804:120:S
        sysbench-3805  [01]   151.412149:   3805:120:S ==>  3797:120:R
        sysbench-3807  [02]   151.412150:   3807:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.412152:   3797:120:R   +  3805:120:S
          mysqld-3798  [02]   151.412157:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412159:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412160:   3806:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.412167:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.412168:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.412169:   3807:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.412171:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.412174:   3808:120:R   +  3800:120:S
          mysqld-3797  [01]   151.412186:   3797:120:S ==>  3805:120:R
        sysbench-3808  [03]   151.412192:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.412196:   3800:120:R   +  3808:120:S
          mysqld-3798  [02]   151.412201:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412203:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412217:   3806:120:S ==>  3799:120:R
          mysqld-3791  [07]   151.412226:   3805:120:S   +  3797:120:S
        sysbench-3805  [01]   151.412234:   3805:120:S ==>  3797:120:R
          mysqld-3791  [07]   151.412252:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.412259:   3802:120:S ==>  3810:120:R
          mysqld-3797  [01]   151.412261:   3797:120:R   +  3805:120:S
        sysbench-3810  [05]   151.412262:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412264:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.412279:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.412281:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.412285:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.412287:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.412292:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412295:   3805:120:R   +  3797:120:S
          mysqld-3800  [03]   151.412301:   3800:120:S ==>  3808:120:R
        sysbench-3805  [01]   151.412304:   3805:120:S ==>  3797:120:R
        sysbench-3808  [03]   151.412307:   3808:120:R   +  3800:120:S
          mysqld-3797  [01]   151.412307:   3797:120:R   +  3805:120:S
        sysbench-3808  [03]   151.412318:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.412321:   3800:120:R   +  3808:120:S
          mysqld-3797  [01]   151.412338:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412340:   3805:120:R   +  3797:120:S
        sysbench-3805  [01]   151.412349:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412352:   3797:120:R   +  3805:120:S
          mysqld-3799  [02]   151.412356:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.412358:   3799:120:R ==>  3807:120:R
          mysqld-3796  [00]   151.412368:   3796:120:S ==>  3804:120:R
        sysbench-3804  [00]   151.412373:   3804:120:S ==>  3801:120:R
          mysqld-3797  [01]   151.412374:   3797:120:R   +  3796:120:S
          mysqld-3801  [00]   151.412378:   3801:120:R   +  3809:120:S
        sysbench-3807  [02]   151.412381:   3807:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.412382:   3797:120:R ==>  3805:120:R
        sysbench-3805  [01]   151.412384:   3805:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.412388:   3798:120:S ==>  3799:120:R
          mysqld-3796  [01]   151.412392:   3799:120:R   +  3798:120:S
          mysqld-3799  [02]   151.412395:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.412398:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.412400:   3807:120:S ==>  3798:120:R
          mysqld-3796  [01]   151.412405:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.412405:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412407:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412408:   3806:120:S ==>  3798:120:R
          mysqld-3801  [00]   151.412421:   3801:120:S ==>  3809:120:R
          mysqld-3796  [01]   151.412427:   3809:120:R   +  3787:120:S
          mysqld-3796  [01]   151.412430:   3796:120:R   +  3801:120:S
          mysqld-3796  [01]   151.412433:   3796:120:S ==>  3804:120:R
          mysqld-3798  [02]   151.412434:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412436:   3798:120:R ==>  3806:120:R
        sysbench-3804  [01]   151.412438:   3804:120:R   +  3796:120:S
        sysbench-3809  [00]   151.412439:   3809:120:S ==>  3787:120:R
        sysbench-3806  [02]   151.412451:   3806:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.412458:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412460:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412461:   3806:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.412469:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.412473:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.412483:   3798:120:R   +  3806:120:S
          mysqld-3787  [00]   151.412485:   3787:120:S ==>     0:140:R
          mysqld-3798  [02]   151.412485:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412495:   3806:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.412503:   3798:120:R   +  3806:120:S
          mysqld-3791  [07]   151.412504:   3791:120:S ==>  3803:120:R
          mysqld-3798  [02]   151.412505:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412506:   3806:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.412511:   3803:120:R   +  3791:120:S
        sysbench-3803  [07]   151.412518:   3803:120:S ==>  3791:120:R
          mysqld-3798  [02]   151.412535:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412537:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412547:   3806:120:S ==>  3799:120:R
          mysqld-3796  [01]   151.412550:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.412553:   3804:120:S ==>  3797:120:R
          mysqld-3802  [05]   151.412556:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412559:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.412561:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.412562:   3797:120:R   +  3796:120:S
        sysbench-3810  [05]   151.412564:   3810:120:R   +  3802:120:S
          mysqld-3797  [01]   151.412568:   3797:120:R   +  3805:120:S
          mysqld-3787  [00]   151.412569:   3787:120:S ==>     0:140:R
        sysbench-3810  [05]   151.412574:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.412576:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.412578:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.412579:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412581:   3805:120:R   +  3797:120:S
          mysqld-3791  [07]   151.412590:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.412592:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.412593:   3803:120:R   +  3791:120:S
        sysbench-3805  [01]   151.412593:   3805:120:S ==>  3797:120:R
        sysbench-3810  [05]   151.412594:   3810:120:R   +  3802:120:S
          mysqld-3797  [01]   151.412596:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.412602:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.412605:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.412623:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.412626:   3800:120:R   +  3808:120:S
          mysqld-3797  [01]   151.412630:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412632:   3805:120:R   +  3797:120:S
        sysbench-3803  [07]   151.412632:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.412634:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.412635:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.412637:   3802:120:R   +  3810:120:S
          mysqld-3799  [02]   151.412640:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.412642:   3799:120:R ==>  3807:120:R
        sysbench-3805  [01]   151.412642:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412646:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.412647:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.412648:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.412649:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.412650:   3810:120:R   +  3802:120:S
        sysbench-3807  [02]   151.412656:   3807:120:S ==>  3798:120:R
        sysbench-3810  [05]   151.412657:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.412657:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.412660:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.412660:   3791:120:R   +  3803:120:S
          mysqld-3798  [02]   151.412665:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412666:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412668:   3806:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.412680:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412682:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.412683:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412686:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.412691:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.412697:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412700:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.412704:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.412705:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.412707:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.412707:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412716:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.412717:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.412720:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.412720:   3791:120:R   +  3803:120:S
          mysqld-3797  [01]   151.412737:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412739:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.412740:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412743:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.412746:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.412753:   3805:120:S ==>  3797:120:R
          mysqld-3791  [07]   151.412754:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.412756:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.412756:   3803:120:R   +  3791:120:S
          mysqld-3797  [01]   151.412757:   3797:120:R   +  3805:120:S
        sysbench-3810  [05]   151.412758:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412766:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.412767:   3810:120:S ==>  3802:120:R
          mysqld-3791  [07]   151.412769:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.412770:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.412790:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412792:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.412793:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412795:   3805:120:R   +  3797:120:S
          mysqld-3798  [02]   151.412799:   3798:120:R   +  3806:120:S
          mysqld-3787  [00]   151.412802:   3787:120:S ==>     0:140:R
          mysqld-3798  [02]   151.412802:   3798:120:R ==>  3806:120:R
          mysqld-3802  [05]   151.412802:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.412802:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.412806:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412806:   3803:120:R   +  3791:120:S
        sysbench-3805  [01]   151.412806:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412809:   3797:120:R   +  3805:120:S
        sysbench-3810  [05]   151.412815:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.412815:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.412818:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.412818:   3791:120:R   +  3803:120:S
        sysbench-3806  [02]   151.412822:   3806:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.412830:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412832:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.412833:   3806:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.412844:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412847:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.412847:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412850:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.412851:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.412853:   3810:120:R   +  3802:120:S
          mysqld-3787  [00]   151.412855:   3787:120:S ==>     0:140:R
          mysqld-3791  [07]   151.412858:   3791:120:S ==>  3803:120:R
        sysbench-3805  [01]   151.412860:   3805:120:S ==>  3797:120:R
        sysbench-3803  [07]   151.412860:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.412863:   3810:120:S ==>  3802:120:R
          mysqld-3797  [01]   151.412864:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.412865:   3800:120:S ==>  3808:120:R
          mysqld-3802  [05]   151.412866:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.412868:   3808:120:R   +  3800:120:S
        sysbench-3803  [07]   151.412870:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.412873:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.412879:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.412882:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.412915:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412916:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.412918:   3800:120:S ==>  3808:120:R
          mysqld-3802  [05]   151.412919:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.412920:   3791:120:S ==>  3803:120:R
        sysbench-3808  [03]   151.412920:   3808:120:R   +  3800:120:S
        sysbench-3810  [05]   151.412922:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412922:   3803:120:R   +  3791:120:S
          mysqld-3787  [00]   151.412926:   3787:120:S ==>     0:140:R
          mysqld-3798  [02]   151.412929:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.412931:   3798:120:R ==>  3806:120:R
        sysbench-3810  [05]   151.412931:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.412932:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.412935:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.412935:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.412935:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.412939:   3800:120:R   +  3808:120:S
        sysbench-3806  [02]   151.412945:   3806:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.412952:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.412953:      0:140:R ==>  3787:120:R
          mysqld-3799  [02]   151.412954:   3799:120:R   +  3807:120:S
          mysqld-3800  [03]   151.412954:   3800:120:S ==>  3808:120:R
          mysqld-3799  [02]   151.412955:   3799:120:R ==>  3807:120:R
        sysbench-3808  [03]   151.412956:   3808:120:R   +  3800:120:S
        sysbench-3807  [02]   151.412956:   3807:120:S ==>  3799:120:R
          mysqld-3787  [00]   151.412959:   3787:120:S ==>     0:140:R
        sysbench-3808  [03]   151.412966:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.412967:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.412967:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.412969:   3800:120:R   +  3808:120:S
        sysbench-3810  [05]   151.412969:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.412969:   3803:120:R   +  3791:120:S
          mysqld-3797  [01]   151.412973:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.412976:   3805:120:R   +  3797:120:S
        sysbench-3803  [07]   151.412978:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.412978:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.412981:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.412981:   3791:120:R   +  3803:120:S
        sysbench-3805  [01]   151.412993:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.412996:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.413010:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413011:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.413012:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.413013:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.413013:   3800:120:S ==>  3808:120:R
        sysbench-3810  [05]   151.413015:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.413016:   3803:120:R   +  3791:120:S
        sysbench-3808  [03]   151.413017:   3808:120:R   +  3800:120:S
          mysqld-3787  [00]   151.413023:   3787:120:S ==>     0:140:R
        sysbench-3810  [05]   151.413025:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.413027:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.413028:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.413029:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.413030:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.413031:   3800:120:R   +  3808:120:S
          mysqld-3791  [07]   151.413039:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.413042:   3803:120:S   +  3791:120:S
        sysbench-3803  [07]   151.413044:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.413063:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413065:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.413065:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.413067:   3791:120:R   +  3803:120:S
        sysbench-3810  [05]   151.413067:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.413071:   3800:120:S ==>  3808:120:R
          mysqld-3787  [00]   151.413072:   3787:120:S ==>     0:140:R
        sysbench-3808  [03]   151.413074:   3808:120:R   +  3800:120:S
          mysqld-3791  [07]   151.413076:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.413077:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.413078:   3803:120:R   +  3791:120:S
          mysqld-3802  [05]   151.413080:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.413083:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413086:   3800:120:R   +  3808:120:S
        sysbench-3803  [07]   151.413087:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.413090:   3791:120:R   +  3803:120:S
          mysqld-3797  [01]   151.413106:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413108:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413110:   3797:120:S ==>  3805:120:R
          mysqld-3802  [05]   151.413111:   3802:120:S ==>  3810:120:R
        sysbench-3805  [01]   151.413112:   3805:120:R   +  3797:120:S
        sysbench-3810  [05]   151.413114:   3810:120:R   +  3802:120:S
          mysqld-3787  [00]   151.413115:   3787:120:S ==>     0:140:R
          mysqld-3791  [07]   151.413121:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.413122:   3800:120:S ==>  3808:120:R
        sysbench-3810  [05]   151.413123:   3810:120:S ==>  3802:120:R
        sysbench-3805  [01]   151.413123:   3805:120:S ==>  3797:120:R
        sysbench-3803  [07]   151.413124:   3803:120:R   +  3791:120:S
        sysbench-3808  [03]   151.413124:   3808:120:R   +  3800:120:S
          mysqld-3802  [05]   151.413126:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.413127:   3797:120:R   +  3805:120:S
        sysbench-3808  [03]   151.413133:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.413133:   3803:120:S ==>  3791:120:R
          mysqld-3800  [03]   151.413136:   3800:120:R   +  3808:120:S
          mysqld-3791  [07]   151.413136:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.413158:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413160:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.413160:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.413163:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.413167:   3800:120:S ==>  3808:120:R
          mysqld-3787  [00]   151.413168:   3787:120:S ==>     0:140:R
          mysqld-3791  [07]   151.413169:   3791:120:S ==>  3803:120:R
        sysbench-3808  [03]   151.413169:   3808:120:R   +  3800:120:S
        sysbench-3803  [07]   151.413172:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.413172:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.413175:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.413178:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.413181:   3803:120:S ==>  3791:120:R
          mysqld-3800  [03]   151.413182:   3800:120:R   +  3808:120:S
          mysqld-3791  [07]   151.413184:   3791:120:R   +  3803:120:S
          mysqld-3799  [02]   151.413213:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.413215:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.413219:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413220:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413222:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413224:   3808:120:R   +  3800:120:S
          mysqld-3787  [00]   151.413228:   3787:120:S ==>     0:140:R
        sysbench-3808  [03]   151.413234:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413237:   3800:120:R   +  3808:120:S
        sysbench-3807  [02]   151.413237:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.413246:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.413247:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.413248:   3806:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.413274:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413275:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413277:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413279:   3808:120:R   +  3800:120:S
          mysqld-3787  [00]   151.413282:   3787:120:S ==>     0:140:R
        sysbench-3808  [03]   151.413289:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413292:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.413300:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413301:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.413303:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.413304:   3791:120:S ==>  3803:120:R
          mysqld-3787  [00]   151.413309:   3787:120:S ==>     0:140:R
        sysbench-3810  [05]   151.413313:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.413315:   3803:120:R   +  3791:120:S
          mysqld-3800  [03]   151.413325:   3800:120:S ==>  3808:120:R
        sysbench-3810  [05]   151.413329:   3810:120:S ==>  3802:120:R
        sysbench-3808  [03]   151.413331:   3808:120:R   +  3800:120:S
        sysbench-3803  [07]   151.413331:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.413333:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.413334:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.413339:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413342:   3800:120:R   +  3808:120:S
          mysqld-3797  [01]   151.413380:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413381:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413383:   3797:120:S ==>  3805:120:R
          mysqld-3800  [03]   151.413384:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413387:   3808:120:R   +  3800:120:S
          mysqld-3787  [00]   151.413390:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413390:   3805:120:R   +  3797:120:S
        sysbench-3808  [03]   151.413396:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413399:   3800:120:R   +  3808:120:S
        sysbench-3805  [01]   151.413408:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.413411:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.413431:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413432:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413434:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413436:   3808:120:R   +  3800:120:S
          mysqld-3791  [07]   151.413438:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.413439:   3802:120:S ==>  3810:120:R
          mysqld-3787  [00]   151.413439:   3787:120:S ==>     0:140:R
        sysbench-3803  [07]   151.413441:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.413442:   3810:120:R   +  3802:120:S
        sysbench-3808  [03]   151.413446:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.413449:   3800:120:R   +  3808:120:S
        sysbench-3810  [05]   151.413452:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.413452:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.413455:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.413455:   3802:120:R   +  3810:120:S
          mysqld-3798  [02]   151.413468:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.413470:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.413482:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413483:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413484:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413486:   3808:120:R   +  3800:120:S
        sysbench-3806  [02]   151.413491:   3806:120:S ==>  3799:120:R
        sysbench-3808  [03]   151.413499:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.413500:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.413502:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.413503:   3800:120:R   +  3808:120:S
        sysbench-3807  [02]   151.413504:   3807:120:S ==>  3799:120:R
          mysqld-3787  [00]   151.413520:   3787:120:S ==>     0:140:R
          mysqld-3800  [03]   151.413632:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413633:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413636:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.413639:   3808:120:R   +  3800:120:S
          mysqld-3797  [01]   151.413640:   3797:120:S ==>  3805:120:R
          mysqld-3787  [00]   151.413641:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413643:   3805:120:R   +  3797:120:S
        sysbench-3805  [01]   151.413654:   3805:120:S ==>  3797:120:R
        sysbench-3808  [03]   151.413656:   3808:120:S ==>  3800:120:R
          mysqld-3797  [01]   151.413657:   3797:120:R   +  3805:120:S
          mysqld-3800  [03]   151.413659:   3800:120:R   +  3808:120:S
          mysqld-3797  [01]   151.413671:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413673:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413674:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.413676:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.413680:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413691:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.413695:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.413696:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.413700:   3803:120:R   +  3791:120:S
          mysqld-3802  [05]   151.413704:   3802:120:S ==>  3810:120:R
          mysqld-3797  [01]   151.413706:   3797:120:S ==>  3805:120:R
        sysbench-3810  [05]   151.413707:   3810:120:R   +  3802:120:S
        sysbench-3805  [01]   151.413708:   3805:120:R   +  3797:120:S
        sysbench-3805  [01]   151.413715:   3805:120:S ==>  3797:120:R
        sysbench-3803  [07]   151.413717:   3803:120:S ==>  3791:120:R
          mysqld-3797  [01]   151.413718:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.413720:   3791:120:R   +  3803:120:S
        sysbench-3810  [05]   151.413723:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.413726:   3802:120:R   +  3810:120:S
          mysqld-3799  [02]   151.413747:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.413749:   3799:120:R ==>  3807:120:R
          mysqld-3800  [03]   151.413754:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413755:      0:140:R ==>  3787:120:R
          mysqld-3800  [03]   151.413757:   3800:120:S ==>  3808:120:R
          mysqld-3797  [01]   151.413757:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.413760:   3805:120:R   +  3797:120:S
        sysbench-3808  [03]   151.413760:   3808:120:R   +  3800:120:S
        sysbench-3807  [02]   151.413761:   3807:120:S ==>  3798:120:R
          mysqld-3787  [00]   151.413764:   3787:120:S ==>     0:140:R
        sysbench-3808  [03]   151.413770:   3808:120:S ==>  3800:120:R
        sysbench-3805  [01]   151.413771:   3805:120:S ==>  3797:120:R
          mysqld-3798  [02]   151.413771:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.413772:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.413773:   3800:120:R   +  3808:120:S
        sysbench-3806  [02]   151.413774:   3806:120:S ==>  3798:120:R
          mysqld-3797  [01]   151.413774:   3797:120:R   +  3805:120:S
          mysqld-3797  [01]   151.413808:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413810:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413811:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.413813:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.413816:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413823:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.413826:   3797:120:R   +  3805:120:S
          mysqld-3797  [01]   151.413857:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413859:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413860:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.413862:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.413865:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413871:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.413874:   3797:120:R   +  3805:120:S
          mysqld-3797  [01]   151.413905:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413907:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413908:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.413910:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.413913:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413919:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.413922:   3797:120:R   +  3805:120:S
          mysqld-3791  [07]   151.413959:   3791:120:S ==>  3803:120:R
          mysqld-3797  [01]   151.413959:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.413961:      0:140:R ==>  3787:120:R
          mysqld-3797  [01]   151.413963:   3803:120:R   +  3791:120:S
          mysqld-3802  [05]   151.413964:   3802:120:S ==>  3810:120:R
          mysqld-3797  [01]   151.413966:   3797:120:S ==>  3805:120:R
        sysbench-3810  [05]   151.413967:   3810:120:R   +  3802:120:S
          mysqld-3787  [00]   151.413968:   3787:120:S ==>     0:140:R
        sysbench-3805  [01]   151.413968:   3805:120:R   +  3797:120:S
        sysbench-3803  [07]   151.413973:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.413977:   3810:120:S ==>  3802:120:R
        sysbench-3805  [01]   151.413979:   3805:120:S ==>  3797:120:R
          mysqld-3791  [07]   151.413979:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.413980:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.413983:   3797:120:R   +  3805:120:S
          mysqld-3798  [02]   151.413997:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.413999:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414014:   3806:120:S ==>  3799:120:R
          mysqld-3791  [07]   151.414023:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414025:      0:140:R ==>  3787:120:R
          mysqld-3791  [07]   151.414029:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.414029:   3802:120:S ==>  3810:120:R
          mysqld-3797  [01]   151.414030:   3797:120:S ==>  3805:120:R
        sysbench-3803  [07]   151.414031:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414032:   3810:120:R   +  3802:120:S
        sysbench-3805  [01]   151.414033:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.414039:   3787:120:S ==>     0:140:R
          mysqld-3800  [03]   151.414042:   3800:120:S ==>  3808:120:R
        sysbench-3805  [01]   151.414045:   3805:120:S ==>  3797:120:R
        sysbench-3808  [03]   151.414045:   3808:120:R   +  3800:120:S
          mysqld-3799  [02]   151.414047:   3799:120:R   +  3807:120:S
          mysqld-3797  [01]   151.414048:   3797:120:R   +  3805:120:S
          mysqld-3799  [02]   151.414049:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414051:   3807:120:S ==>  3799:120:R
        sysbench-3810  [05]   151.414053:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.414056:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.414057:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.414059:   3791:120:R   +  3803:120:S
          mysqld-3799  [02]   151.414061:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414062:   3799:120:R ==>  3807:120:R
        sysbench-3808  [03]   151.414064:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.414067:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.414068:   3802:120:S ==>  3810:120:R
          mysqld-3800  [03]   151.414069:   3800:120:R ==>  3808:120:R
          mysqld-3791  [07]   151.414069:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.414070:   3810:120:R   +  3802:120:S
        sysbench-3808  [03]   151.414071:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.414071:   3803:120:R   +  3791:120:S
          mysqld-3797  [01]   151.414077:   3797:120:S ==>  3805:120:R
        sysbench-3810  [05]   151.414078:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.414078:   3803:120:S ==>  3791:120:R
        sysbench-3805  [01]   151.414080:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.414081:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.414081:   3791:120:R   +  3803:120:S
        sysbench-3807  [02]   151.414084:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.414090:   3799:120:R   +  3807:120:S
        sysbench-3805  [01]   151.414091:   3805:120:S ==>  3797:120:R
          mysqld-3799  [02]   151.414092:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414093:   3807:120:S ==>  3799:120:R
          mysqld-3797  [01]   151.414095:   3797:120:R   +  3805:120:S
          mysqld-3799  [02]   151.414105:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414106:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414113:   3807:120:S ==>  3798:120:R
          mysqld-3791  [07]   151.414122:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.414143:   3803:120:S ==>     0:140:R
          mysqld-3802  [05]   151.414167:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414169:      0:140:R ==>  3787:120:R
          mysqld-3802  [05]   151.414170:      0:140:R   +  3791:120:S
          mysqld-3797  [01]   151.414173:   3797:120:S ==>  3805:120:R
          <idle>-0     [07]   151.414174:      0:140:R ==>  3791:120:R
          mysqld-3798  [02]   151.414175:   3800:120:S   +  3800:120:S
        sysbench-3805  [01]   151.414175:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.414176:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.414178:   3810:120:R   +  3802:120:S
          mysqld-3787  [00]   151.414179:   3787:120:S ==>     0:140:R
          mysqld-3791  [07]   151.414181:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.414182:   3800:120:R ==>  3796:120:R
          mysqld-3798  [02]   151.414184:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414186:   3798:120:R ==>  3806:120:R
        sysbench-3805  [01]   151.414187:   3805:120:S ==>  3797:120:R
        sysbench-3806  [02]   151.414189:   3806:120:S ==>  3799:120:R
          mysqld-3797  [01]   151.414191:   3797:120:R   +  3805:120:S
        sysbench-3810  [05]   151.414191:   3810:120:S ==>  3802:120:R
          mysqld-3799  [02]   151.414194:   3799:120:R   +  3807:120:S
          mysqld-3802  [05]   151.414194:   3802:120:R   +  3810:120:S
          mysqld-3799  [02]   151.414195:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414196:   3807:120:S ==>  3799:120:R
          mysqld-3791  [07]   151.414217:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414218:      0:140:R ==>  3787:120:R
          mysqld-3791  [07]   151.414219:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.414222:   3803:120:R   +  3791:120:S
          mysqld-3797  [01]   151.414222:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.414225:   3805:120:R   +  3797:120:S
          mysqld-3787  [00]   151.414226:   3787:120:S ==>     0:140:R
          mysqld-3802  [05]   151.414229:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.414231:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.414232:   3803:120:S ==>  3791:120:R
          mysqld-3799  [02]   151.414233:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414235:   3799:120:R ==>  3807:120:R
          mysqld-3791  [07]   151.414235:   3791:120:R   +  3803:120:S
        sysbench-3805  [01]   151.414237:   3805:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.414240:   3797:120:R   +  3805:120:S
        sysbench-3810  [05]   151.414241:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.414244:   3802:120:R   +  3810:120:S
        sysbench-3807  [02]   151.414247:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.414255:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414257:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414258:   3807:120:S ==>  3799:120:R
          mysqld-3791  [07]   151.414270:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414272:      0:140:R ==>  3787:120:R
          mysqld-3791  [07]   151.414273:   3791:120:S ==>  3803:120:R
          mysqld-3797  [01]   151.414274:   3797:120:S ==>  3805:120:R
          mysqld-3802  [05]   151.414275:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.414276:   3803:120:R   +  3791:120:S
        sysbench-3805  [01]   151.414277:   3805:120:R   +  3797:120:S
        sysbench-3810  [05]   151.414278:   3810:120:R   +  3802:120:S
          mysqld-3787  [00]   151.414279:   3787:120:S ==>     0:140:R
        sysbench-3803  [07]   151.414285:   3803:120:S ==>  3791:120:R
          mysqld-3799  [02]   151.414287:   3799:120:R   +  3807:120:S
        sysbench-3810  [05]   151.414288:   3810:120:S ==>  3802:120:R
          mysqld-3799  [02]   151.414288:   3799:120:R ==>  3807:120:R
          mysqld-3791  [07]   151.414289:   3791:120:R   +  3803:120:S
        sysbench-3805  [01]   151.414289:   3805:120:S ==>  3797:120:R
          mysqld-3802  [05]   151.414291:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.414292:   3797:120:R   +  3805:120:S
        sysbench-3807  [02]   151.414300:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.414309:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414310:   3798:120:R ==>  3806:120:R
          mysqld-3791  [07]   151.414322:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.414323:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414325:      0:140:R ==>  3787:120:R
        sysbench-3806  [02]   151.414327:   3806:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.414331:   3803:120:S   +  3791:120:S
          mysqld-3787  [00]   151.414333:   3787:120:S ==>     0:140:R
        sysbench-3803  [07]   151.414333:   3803:120:S ==>  3791:120:R
          mysqld-3798  [02]   151.414334:   3798:120:R   +  3806:120:S
          mysqld-3802  [05]   151.414335:   3802:120:S ==>  3810:120:R
          mysqld-3798  [02]   151.414335:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414337:   3806:120:S ==>  3799:120:R
        sysbench-3810  [05]   151.414337:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.414343:   3791:120:R   +  3803:120:S
          mysqld-3799  [02]   151.414343:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414345:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414346:   3807:120:S ==>  3798:120:R
        sysbench-3810  [05]   151.414346:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.414349:   3802:120:R   +  3810:120:S
          mysqld-3796  [03]   151.414351:   3797:120:R   +  3804:120:S
          mysqld-3798  [02]   151.414354:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414354:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414362:   3806:120:S ==>  3799:120:R
          mysqld-3796  [03]   151.414366:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414367:      0:140:R ==>  3787:120:R
          mysqld-3796  [03]   151.414369:   3796:120:S ==>  3800:120:R
          mysqld-3787  [00]   151.414374:   3787:120:S ==>     0:140:R
          mysqld-3791  [07]   151.414379:   3791:120:S ==>  3803:120:R
          mysqld-3802  [05]   151.414380:   3802:120:S ==>  3810:120:R
        sysbench-3803  [07]   151.414381:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414382:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.414390:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.414391:   3810:120:S ==>  3802:120:R
          mysqld-3799  [02]   151.414391:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414393:   3799:120:R ==>  3807:120:R
          mysqld-3791  [07]   151.414393:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.414394:   3802:120:R   +  3810:120:S
          mysqld-3797  [01]   151.414399:   3797:120:S ==>  3804:120:R
        sysbench-3807  [02]   151.414403:   3807:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.414405:   3800:120:R   +  3796:120:S
          mysqld-3798  [02]   151.414408:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414410:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414411:   3806:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.414418:   3804:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.414421:   3805:120:R   +  3797:120:S
          mysqld-3802  [05]   151.414425:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.414425:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.414427:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.414427:   3803:120:R   +  3791:120:S
        sysbench-3805  [01]   151.414435:   3805:120:S ==>  3797:120:R
        sysbench-3810  [05]   151.414436:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.414436:   3803:120:S ==>  3791:120:R
          mysqld-3797  [01]   151.414439:   3797:120:R   +  3805:120:S
          mysqld-3802  [05]   151.414439:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.414439:   3791:120:R   +  3803:120:S
          mysqld-3798  [02]   151.414443:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414444:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414455:   3806:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.414463:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414464:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414465:   3807:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.414477:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.414478:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.414480:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.414481:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414488:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.414489:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.414491:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.414492:   3791:120:R   +  3803:120:S
          mysqld-3799  [02]   151.414494:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414495:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414506:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.414514:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414515:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414516:   3806:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.414521:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414523:   3800:120:R ==>  3808:120:R
          mysqld-3802  [05]   151.414523:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.414524:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.414525:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.414526:   3803:120:R   +  3791:120:S
          mysqld-3797  [01]   151.414528:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.414531:   3805:120:R   +  3797:120:S
        sysbench-3810  [05]   151.414534:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.414535:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.414536:   3808:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.414537:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.414538:   3791:120:R   +  3803:120:S
          mysqld-3796  [03]   151.414540:   3805:120:R   +  3804:120:S
          mysqld-3798  [02]   151.414542:   3798:120:R   +  3806:120:S
        sysbench-3805  [01]   151.414542:   3805:120:S ==>  3804:120:R
          mysqld-3798  [02]   151.414544:   3798:120:R ==>  3806:120:R
        sysbench-3804  [01]   151.414545:   3804:120:S ==>  3797:120:R
          mysqld-3797  [01]   151.414548:   3797:120:R   +  3805:120:S
          mysqld-3796  [03]   151.414553:   3797:120:R   +  3804:120:S
        sysbench-3806  [02]   151.414556:   3806:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.414564:   3799:120:R   +  3807:120:S
          mysqld-3796  [03]   151.414564:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414565:      0:140:R ==>  3787:120:R
          mysqld-3799  [02]   151.414565:   3799:120:R ==>  3807:120:R
          mysqld-3796  [03]   151.414566:   3796:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.414566:   3807:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.414570:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.414572:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.414575:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.414577:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414583:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.414587:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.414587:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.414592:   3791:120:R   +  3803:120:S
          mysqld-3799  [02]   151.414592:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414594:   3799:120:R ==>  3807:120:R
          mysqld-3787  [00]   151.414603:   3787:120:S ==>     0:140:R
        sysbench-3807  [02]   151.414605:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.414612:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414614:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414616:   3806:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.414619:   3802:120:S ==>  3810:120:R
          mysqld-3800  [03]   151.414620:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414622:   3800:120:R ==>  3808:120:R
        sysbench-3810  [05]   151.414622:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.414623:   3791:120:S ==>  3803:120:R
        sysbench-3808  [03]   151.414623:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.414625:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414630:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.414630:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414632:   3800:120:R ==>  3808:120:R
          mysqld-3802  [05]   151.414634:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.414634:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.414637:   3791:120:R   +  3803:120:S
          mysqld-3798  [02]   151.414642:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414643:   3798:120:R ==>  3806:120:R
        sysbench-3808  [03]   151.414652:   3808:120:S ==>  3800:120:R
        sysbench-3806  [02]   151.414656:   3806:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.414658:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414660:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414661:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.414663:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414664:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414666:   3807:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.414667:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414668:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414676:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.414681:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414682:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414683:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.414688:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414689:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414699:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.414706:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414707:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414708:   3806:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.414713:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414715:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414726:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.414731:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414732:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.414734:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414735:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414737:   3808:120:S ==>  3800:120:R
        sysbench-3806  [02]   151.414741:   3806:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.414745:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.414747:   3810:120:R   +  3802:120:S
          mysqld-3799  [02]   151.414749:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414750:   3799:120:R ==>  3807:120:R
          mysqld-3791  [07]   151.414751:   3791:120:S ==>  3803:120:R
        sysbench-3807  [02]   151.414752:   3807:120:S ==>  3799:120:R
        sysbench-3803  [07]   151.414754:   3803:120:R   +  3791:120:S
          mysqld-3800  [03]   151.414760:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414762:   3800:120:R ==>  3808:120:R
        sysbench-3810  [05]   151.414763:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.414766:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.414770:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.414772:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.414773:   3791:120:R   +  3803:120:S
          mysqld-3797  [01]   151.414775:   3797:120:S ==>  3804:120:R
          mysqld-3799  [02]   151.414777:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414778:   3799:120:R ==>  3807:120:R
        sysbench-3804  [01]   151.414779:   3800:120:R   +  3796:120:S
          mysqld-3800  [03]   151.414781:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414783:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414784:   3808:120:S ==>  3796:120:R
          mysqld-3796  [03]   151.414786:   3796:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.414789:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.414796:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414798:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414799:   3806:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.414801:   3800:120:R   +  3796:120:S
        sysbench-3804  [01]   151.414803:   3804:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.414806:   3805:120:R   +  3797:120:S
          mysqld-3800  [03]   151.414810:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414811:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414820:   3808:120:S ==>  3796:120:R
        sysbench-3805  [01]   151.414821:   3805:120:S ==>  3797:120:R
          mysqld-3798  [02]   151.414822:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414823:   3798:120:R ==>  3806:120:R
          mysqld-3796  [03]   151.414825:   3797:120:R   +  3804:120:S
          mysqld-3797  [01]   151.414825:   3797:120:R   +  3805:120:S
        sysbench-3806  [02]   151.414834:   3806:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.414841:   3799:120:R   +  3807:120:S
          mysqld-3796  [03]   151.414841:      0:140:R   +  3787:120:S
          <idle>-0     [00]   151.414842:      0:140:R ==>  3787:120:R
          mysqld-3799  [02]   151.414842:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414843:   3807:120:S ==>  3799:120:R
          mysqld-3796  [03]   151.414844:   3796:120:S ==>  3800:120:R
          mysqld-3787  [00]   151.414850:   3787:120:S ==>     0:140:R
          mysqld-3800  [03]   151.414850:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414853:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414854:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.414858:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.414861:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.414865:   3791:120:S ==>  3803:120:R
          mysqld-3799  [02]   151.414866:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414868:   3799:120:R ==>  3807:120:R
        sysbench-3803  [07]   151.414868:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.414872:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.414876:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.414877:   3803:120:S ==>  3791:120:R
        sysbench-3807  [02]   151.414877:   3807:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.414879:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414881:   3800:120:R ==>  3808:120:R
          mysqld-3791  [07]   151.414881:   3791:120:R   +  3803:120:S
          mysqld-3798  [02]   151.414885:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414886:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414888:   3806:120:S ==>  3798:120:R
        sysbench-3808  [03]   151.414891:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.414899:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414900:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414901:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.414914:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414915:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.414924:   3800:120:R   +  3808:120:S
        sysbench-3806  [02]   151.414925:   3806:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.414926:   3800:120:R ==>  3808:120:R
          mysqld-3799  [02]   151.414932:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414933:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414934:   3807:120:S ==>  3799:120:R
        sysbench-3808  [03]   151.414936:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.414943:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414945:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414946:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.414958:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.414959:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.414969:   3807:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.414969:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414971:   3800:120:R ==>  3808:120:R
          mysqld-3798  [02]   151.414976:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.414977:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.414979:   3806:120:S ==>  3798:120:R
        sysbench-3808  [03]   151.414981:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.414989:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.414990:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.414992:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.415005:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415006:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.415014:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415015:   3800:120:R ==>  3808:120:R
        sysbench-3806  [02]   151.415016:   3806:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.415023:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415024:   3799:120:R ==>  3807:120:R
        sysbench-3808  [03]   151.415025:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.415026:   3807:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.415032:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415034:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415036:   3808:120:S ==>  3800:120:R
          mysqld-3797  [01]   151.415046:   3797:120:S ==>  3805:120:R
        sysbench-3805  [01]   151.415048:   3805:120:R   +  3797:120:S
          mysqld-3799  [02]   151.415051:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415053:   3799:120:R ==>  3807:120:R
        sysbench-3805  [01]   151.415058:   3805:120:S ==>  3804:120:R
          mysqld-3800  [03]   151.415059:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415061:   3800:120:R ==>  3808:120:R
        sysbench-3807  [02]   151.415063:   3807:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.415063:   3804:120:R   +  3796:120:S
          mysqld-3798  [02]   151.415070:   3798:120:R   +  3806:120:S
        sysbench-3804  [01]   151.415071:   3804:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.415072:   3798:120:R ==>  3806:120:R
        sysbench-3808  [03]   151.415072:   3808:120:S ==>  3800:120:R
        sysbench-3806  [02]   151.415073:   3806:120:S ==>  3798:120:R
          mysqld-3796  [01]   151.415075:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.415080:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415081:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415082:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.415100:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415102:   3798:120:R ==>  3806:120:R
          mysqld-3800  [03]   151.415107:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415108:   3800:120:R ==>  3808:120:R
        sysbench-3806  [02]   151.415112:   3806:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.415114:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415114:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.415117:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415117:   3803:120:R   +  3791:120:S
        sysbench-3808  [03]   151.415118:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.415121:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415122:   3799:120:R ==>  3807:120:R
          mysqld-3796  [01]   151.415123:   3796:120:S ==>  3804:120:R
        sysbench-3807  [02]   151.415124:   3807:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.415125:   3800:120:R   +  3808:120:S
        sysbench-3804  [01]   151.415126:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.415126:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415128:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.415133:   3803:120:S ==>  3791:120:R
        sysbench-3810  [05]   151.415133:   3810:120:S ==>  3802:120:R
        sysbench-3804  [01]   151.415135:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.415137:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415137:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.415138:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.415153:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415155:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415164:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.415171:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415173:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415174:   3808:120:S ==>  3800:120:R
          mysqld-3796  [01]   151.415177:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415179:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.415188:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.415191:   3796:120:R   +  3804:120:S
          mysqld-3796  [01]   151.415221:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415223:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.415231:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.415234:   3796:120:R   +  3804:120:S
          mysqld-3799  [02]   151.415246:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415248:   3799:120:R ==>  3807:120:R
          mysqld-3796  [01]   151.415263:   3796:120:S ==>  3804:120:R
        sysbench-3807  [02]   151.415266:   3807:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.415266:   3804:120:R   +  3796:120:S
          mysqld-3798  [02]   151.415274:   3798:120:R   +  3806:120:S
        sysbench-3804  [01]   151.415275:   3804:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.415276:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.415277:   3806:120:S ==>  3798:120:R
          mysqld-3796  [01]   151.415278:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.415283:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415285:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415302:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.415303:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415305:   3798:120:R ==>  3806:120:R
          mysqld-3796  [01]   151.415309:   3796:120:S ==>  3804:120:R
          mysqld-3800  [03]   151.415309:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415311:   3800:120:R ==>  3808:120:R
        sysbench-3804  [01]   151.415312:   3804:120:R   +  3796:120:S
        sysbench-3808  [03]   151.415312:   3808:120:S ==>  3800:120:R
        sysbench-3806  [02]   151.415315:   3806:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.415320:   3804:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.415323:   3798:120:R   +  3806:120:S
          mysqld-3796  [01]   151.415324:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.415325:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.415326:   3806:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.415352:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415353:   3798:120:R ==>  3806:120:R
          mysqld-3796  [01]   151.415357:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415360:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.415361:   3802:120:S ==>  3810:120:R
        sysbench-3806  [02]   151.415363:   3806:120:S ==>  3798:120:R
          mysqld-3791  [07]   151.415365:   3791:120:S ==>  3803:120:R
        sysbench-3804  [01]   151.415368:   3804:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.415371:   3798:120:R   +  3806:120:S
          mysqld-3796  [01]   151.415371:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.415372:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.415374:   3806:120:S ==>  3798:120:R
        sysbench-3810  [05]   151.415374:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415376:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415383:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.415386:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.415387:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415389:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.415400:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.415402:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415402:   3791:120:S ==>  3803:120:R
          mysqld-3796  [01]   151.415403:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.415404:   3803:120:R   +  3791:120:S
          mysqld-3800  [03]   151.415407:   3800:120:R   +  3808:120:S
        sysbench-3804  [01]   151.415411:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.415419:   3804:120:S ==>  3796:120:R
          mysqld-3800  [03]   151.415421:   3800:120:S ==>  3808:120:R
          mysqld-3796  [01]   151.415422:   3796:120:R   +  3804:120:S
        sysbench-3808  [03]   151.415424:   3808:120:R   +  3800:120:S
        sysbench-3810  [05]   151.415427:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415430:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415430:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.415433:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.415433:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.415436:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415437:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.415439:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.415441:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415443:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.415443:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415445:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415450:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.415451:   3803:120:S ==>  3791:120:R
          mysqld-3796  [01]   151.415452:   3796:120:S ==>  3804:120:R
          mysqld-3802  [05]   151.415453:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415454:   3791:120:R   +  3803:120:S
        sysbench-3804  [01]   151.415454:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.415463:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.415466:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.415475:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415477:   3798:120:R ==>  3806:120:R
          mysqld-3802  [05]   151.415492:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415493:   3791:120:S ==>  3803:120:R
        sysbench-3806  [02]   151.415494:   3806:120:S ==>  3799:120:R
        sysbench-3810  [05]   151.415495:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415495:   3803:120:R   +  3791:120:S
          mysqld-3796  [01]   151.415499:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415501:   3804:120:R   +  3796:120:S
          mysqld-3799  [02]   151.415502:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415504:   3799:120:R ==>  3807:120:R
        sysbench-3810  [05]   151.415505:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.415505:   3803:120:S ==>  3791:120:R
        sysbench-3807  [02]   151.415506:   3807:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.415508:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415508:   3791:120:R   +  3803:120:S
        sysbench-3804  [01]   151.415510:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.415513:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.415541:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415542:   3791:120:S ==>  3803:120:R
          mysqld-3796  [01]   151.415543:   3796:120:S ==>  3804:120:R
        sysbench-3810  [05]   151.415543:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415544:   3803:120:R   +  3791:120:S
        sysbench-3804  [01]   151.415545:   3804:120:R   +  3796:120:S
        sysbench-3810  [05]   151.415552:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.415553:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.415554:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.415556:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415556:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.415557:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.415587:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415588:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.415590:   3810:120:R   +  3802:120:S
        sysbench-3803  [07]   151.415590:   3803:120:R   +  3791:120:S
          mysqld-3799  [02]   151.415591:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.415592:   3799:120:R ==>  3807:120:R
        sysbench-3810  [05]   151.415598:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.415599:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.415601:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415602:   3791:120:R   +  3803:120:S
        sysbench-3807  [02]   151.415603:   3807:120:S ==>  3798:120:R
          mysqld-3798  [02]   151.415611:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415612:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.415613:   3806:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.415631:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.415633:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415634:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.415636:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415641:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415644:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415645:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.415648:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.415660:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415662:   3800:120:R ==>  3808:120:R
          mysqld-3796  [01]   151.415667:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415670:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.415675:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.415677:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415678:   3791:120:S ==>  3803:120:R
        sysbench-3808  [03]   151.415680:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.415680:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415686:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.415688:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415689:   3800:120:R ==>  3808:120:R
          mysqld-3802  [05]   151.415689:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415689:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.415690:   3804:120:S ==>  3796:120:R
        sysbench-3808  [03]   151.415690:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.415692:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.415693:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.415702:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.415704:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.415714:   3806:120:S ==>  3799:120:R
          mysqld-3802  [05]   151.415721:   3802:120:S ==>  3810:120:R
          mysqld-3799  [02]   151.415722:   3799:120:R   +  3807:120:S
        sysbench-3810  [05]   151.415723:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415724:   3791:120:S ==>  3803:120:R
          mysqld-3799  [02]   151.415724:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.415725:   3807:120:S ==>  3799:120:R
        sysbench-3803  [07]   151.415727:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415732:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415735:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415735:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.415738:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.415764:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.415767:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415768:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.415770:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415775:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415778:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415778:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.415781:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.415784:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.415786:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.415796:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.415799:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.415808:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.415810:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.415811:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.415814:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.415818:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415821:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415822:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.415825:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.415845:   3796:120:R ==>  3804:120:R
        sysbench-3804  [01]   151.415847:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.415851:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.415852:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.415854:   3810:120:R   +  3802:120:S
        sysbench-3810  [05]   151.415863:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415867:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.415888:   3803:120:S ==>     0:140:R
          mysqld-3796  [01]   151.415893:      0:140:R   +  3791:120:S
          mysqld-3802  [05]   151.415895:   3802:120:S ==>  3810:120:R
          <idle>-0     [07]   151.415896:      0:140:R ==>  3791:120:R
        sysbench-3810  [05]   151.415898:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.415902:   3800:120:R   +  3808:120:S
        sysbench-3810  [05]   151.415906:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.415910:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.415911:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.415919:   3791:120:S ==>  3803:120:R
          mysqld-3800  [03]   151.415921:   3800:120:S ==>  3808:120:R
        sysbench-3803  [07]   151.415922:   3803:120:R   +  3791:120:S
        sysbench-3808  [03]   151.415924:   3808:120:R   +  3800:120:S
        sysbench-3803  [07]   151.415931:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.415934:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.415934:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.415937:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.415949:   3800:120:S ==>  3808:120:R
          mysqld-3799  [02]   151.415950:   3799:120:R   +  3807:120:S
        sysbench-3808  [03]   151.415952:   3808:120:R   +  3800:120:S
          mysqld-3799  [02]   151.415960:   3799:120:S ==>  3807:120:R
        sysbench-3807  [02]   151.415963:   3807:120:R   +  3799:120:S
          mysqld-3791  [07]   151.415965:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.415967:   3803:120:R   +  3791:120:S
        sysbench-3808  [03]   151.415969:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.415972:   3800:120:R   +  3808:120:S
        sysbench-3803  [07]   151.415976:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.415979:   3791:120:R   +  3803:120:S
        sysbench-3807  [02]   151.415982:   3807:120:S ==>  3799:120:R
          mysqld-3800  [03]   151.415983:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.415985:   3808:120:R   +  3800:120:S
          mysqld-3799  [02]   151.415985:   3799:120:R   +  3807:120:S
        sysbench-3808  [03]   151.415992:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.415994:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.416020:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416023:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.416034:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.416037:   3808:120:R   +  3800:120:S
        sysbench-3810  [05]   151.416038:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416041:   3802:120:R   +  3810:120:S
        sysbench-3808  [03]   151.416046:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.416049:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416079:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.416081:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.416090:   3808:120:S ==>  3800:120:R
          mysqld-3796  [01]   151.416092:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.416093:   3800:120:R   +  3808:120:S
          mysqld-3791  [07]   151.416099:   3791:120:S ==>  3803:120:R
          mysqld-3796  [01]   151.416101:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.416102:   3803:120:R   +  3791:120:S
        sysbench-3804  [01]   151.416104:   3804:120:R   +  3796:120:S
        sysbench-3803  [07]   151.416117:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.416120:   3804:120:S ==>  3796:120:R
          mysqld-3791  [07]   151.416121:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.416122:   3800:120:S ==>  3808:120:R
          mysqld-3796  [01]   151.416124:   3796:120:R   +  3804:120:S
        sysbench-3808  [03]   151.416125:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.416133:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.416136:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.416143:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416146:   3810:120:R   +  3802:120:S
        sysbench-3810  [05]   151.416155:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416158:   3802:120:R   +  3810:120:S
          mysqld-3800  [03]   151.416165:   3800:120:S ==>  3808:120:R
        sysbench-3808  [03]   151.416168:   3808:120:R   +  3800:120:S
        sysbench-3808  [03]   151.416176:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.416179:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416205:   3800:120:S ==>  3808:120:R
          mysqld-3791  [07]   151.416218:   3791:120:S ==>  3803:120:R
          mysqld-3799  [02]   151.416225:   3799:120:S ==>  3807:120:R
        sysbench-3803  [07]   151.416235:   3803:120:R   +  3791:120:S
        sysbench-3803  [07]   151.416244:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.416248:   3791:120:R   +  3803:120:S
        sysbench-3807  [02]   151.416249:   3807:120:R   +  3799:120:S
        sysbench-3808  [03]   151.416252:   3808:120:S ==>  3800:120:R
          mysqld-3802  [05]   151.416253:   3808:120:S   +  3800:120:S
          mysqld-3800  [03]   151.416257:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416258:   3800:120:R ==>  3808:120:R
        sysbench-3807  [02]   151.416260:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416263:   3799:120:R   +  3807:120:S
        sysbench-3808  [03]   151.416269:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.416276:   3800:120:R   +  3808:120:S
          mysqld-3799  [02]   151.416277:   3799:120:S ==>  3807:120:R
          mysqld-3800  [03]   151.416278:   3800:120:R ==>  3808:120:R
        sysbench-3807  [02]   151.416279:   3807:120:R   +  3799:120:S
        sysbench-3808  [03]   151.416279:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.416294:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416297:   3799:120:R   +  3807:120:S
          mysqld-3800  [03]   151.416304:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416305:   3800:120:R ==>  3808:120:R
          mysqld-3799  [02]   151.416307:   3799:120:S ==>  3807:120:R
        sysbench-3807  [02]   151.416308:   3807:120:R   +  3799:120:S
        sysbench-3808  [03]   151.416315:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.416315:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416318:   3799:120:R   +  3807:120:S
          mysqld-3800  [03]   151.416322:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416324:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416325:   3808:120:S ==>  3800:120:R
          mysqld-3800  [03]   151.416348:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416350:   3800:120:R ==>  3808:120:R
          mysqld-3799  [02]   151.416359:   3799:120:S ==>  3807:120:R
        sysbench-3808  [03]   151.416360:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.416362:   3807:120:R   +  3799:120:S
          mysqld-3800  [03]   151.416367:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416368:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416369:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.416371:   3807:120:S ==>  3799:120:R
          mysqld-3796  [01]   151.416373:   3796:120:S ==>  3804:120:R
          mysqld-3799  [02]   151.416374:   3799:120:R   +  3807:120:S
        sysbench-3804  [01]   151.416376:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.416389:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416392:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.416393:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416394:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416404:   3808:120:S ==>  3800:120:R
          mysqld-3796  [01]   151.416405:   3796:120:S ==>  3804:120:R
          mysqld-3799  [02]   151.416410:   3799:120:S ==>  3807:120:R
        sysbench-3804  [01]   151.416410:   3804:120:R   +  3796:120:S
        sysbench-3807  [02]   151.416412:   3807:120:R   +  3799:120:S
          mysqld-3800  [03]   151.416415:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416417:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416418:   3808:120:S ==>  3800:120:R
        sysbench-3807  [02]   151.416421:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416424:   3799:120:R   +  3807:120:S
        sysbench-3804  [01]   151.416426:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416429:   3796:120:R   +  3804:120:S
          mysqld-3796  [01]   151.416440:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416442:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.416443:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416445:   3800:120:R ==>  3808:120:R
        sysbench-3804  [01]   151.416449:   3804:120:S ==>  3796:120:R
          mysqld-3802  [05]   151.416451:   3802:120:S ==>  3810:120:R
          mysqld-3796  [01]   151.416452:   3796:120:R   +  3804:120:S
        sysbench-3808  [03]   151.416455:   3808:120:S ==>  3800:120:R
          mysqld-3799  [02]   151.416457:   3799:120:S ==>  3807:120:R
        sysbench-3810  [05]   151.416459:   3810:120:R   +  3802:120:S
        sysbench-3807  [02]   151.416459:   3807:120:R   +  3799:120:S
          mysqld-3800  [03]   151.416463:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416464:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416465:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.416467:   3791:120:S ==>  3803:120:R
        sysbench-3807  [02]   151.416468:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416471:   3799:120:R   +  3807:120:S
        sysbench-3803  [07]   151.416474:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.416475:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416478:   3802:120:R   +  3810:120:S
          mysqld-3796  [01]   151.416489:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.416490:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.416491:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.416492:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416493:   3800:120:R ==>  3808:120:R
          mysqld-3791  [07]   151.416493:   3791:120:R   +  3803:120:S
        sysbench-3804  [01]   151.416501:   3804:120:S ==>  3796:120:R
        sysbench-3808  [03]   151.416503:   3808:120:S ==>  3800:120:R
          mysqld-3796  [01]   151.416504:   3796:120:R   +  3804:120:S
          mysqld-3799  [02]   151.416504:   3799:120:S ==>  3807:120:R
        sysbench-3807  [02]   151.416507:   3807:120:R   +  3799:120:S
        sysbench-3807  [02]   151.416515:   3807:120:S ==>  3799:120:R
          mysqld-3799  [02]   151.416518:   3799:120:R   +  3807:120:S
          mysqld-3799  [02]   151.416519:   3799:120:R ==>  3807:120:R
        sysbench-3807  [02]   151.416521:   3807:120:S ==>  3798:120:R
          mysqld-3800  [03]   151.416523:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416524:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416526:   3808:120:S ==>  3800:120:R
          mysqld-3798  [02]   151.416528:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.416529:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.416531:   3806:120:S ==>  3798:120:R
          mysqld-3796  [01]   151.416535:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416538:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.416546:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416549:   3796:120:R   +  3804:120:S
          mysqld-3796  [01]   151.416580:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416582:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.416590:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416594:   3796:120:R   +  3804:120:S
          mysqld-3796  [01]   151.416621:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416623:   3804:120:R   +  3796:120:S
        sysbench-3804  [01]   151.416632:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416635:   3796:120:R   +  3804:120:S
          mysqld-3800  [03]   151.416645:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416646:   3800:120:R ==>  3808:120:R
          mysqld-3796  [01]   151.416662:   3796:120:S ==>  3804:120:R
        sysbench-3808  [03]   151.416663:   3808:120:S ==>  3800:120:R
        sysbench-3804  [01]   151.416664:   3804:120:R   +  3796:120:S
          mysqld-3800  [03]   151.416670:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416672:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.416673:   3808:120:S ==>  3800:120:R
        sysbench-3804  [01]   151.416673:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416676:   3796:120:R   +  3804:120:S
          mysqld-3796  [01]   151.416706:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416709:   3804:120:R   +  3796:120:S
          mysqld-3802  [05]   151.416709:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416712:   3810:120:R   +  3802:120:S
        sysbench-3804  [01]   151.416717:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416720:   3796:120:R   +  3804:120:S
          mysqld-3791  [07]   151.416721:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.416722:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.416724:   3803:120:R   +  3791:120:S
          mysqld-3802  [05]   151.416725:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.416733:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.416736:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.416737:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416739:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.416750:   3791:120:S ==>  3803:120:R
          mysqld-3796  [01]   151.416750:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.416752:   3803:120:R   +  3791:120:S
        sysbench-3804  [01]   151.416752:   3804:120:R   +  3796:120:S
        sysbench-3810  [05]   151.416754:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.416757:   3800:120:R   +  3808:120:S
          mysqld-3802  [05]   151.416757:   3802:120:R   +  3810:120:S
          mysqld-3800  [03]   151.416759:   3800:120:R ==>  3808:120:R
        sysbench-3804  [01]   151.416761:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416764:   3796:120:R   +  3804:120:S
        sysbench-3803  [07]   151.416766:   3803:120:S ==>  3791:120:R
          mysqld-3802  [05]   151.416768:   3802:120:S ==>  3810:120:R
          mysqld-3791  [07]   151.416769:   3791:120:R   +  3803:120:S
        sysbench-3808  [03]   151.416769:   3808:120:S ==>  3800:120:R
        sysbench-3810  [05]   151.416770:   3810:120:R   +  3802:120:S
        sysbench-3810  [05]   151.416777:   3810:120:S ==>  3802:120:R
          mysqld-3800  [03]   151.416777:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416779:   3800:120:R ==>  3808:120:R
          mysqld-3802  [05]   151.416780:   3802:120:R   +  3810:120:S
          mysqld-3791  [07]   151.416780:   3791:120:S ==>  3803:120:R
        sysbench-3808  [03]   151.416781:   3808:120:S ==>  3800:120:R
        sysbench-3803  [07]   151.416782:   3803:120:R   +  3791:120:S
          mysqld-3798  [02]   151.416783:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.416785:   3798:120:R ==>  3806:120:R
        sysbench-3803  [07]   151.416789:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.416792:   3791:120:R   +  3803:120:S
          mysqld-3796  [01]   151.416794:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.416796:   3804:120:R   +  3796:120:S
        sysbench-3806  [02]   151.416804:   3806:120:S ==>  3798:120:R
        sysbench-3804  [01]   151.416805:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416808:   3796:120:R   +  3804:120:S
          mysqld-3798  [02]   151.416812:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.416813:   3798:120:R ==>  3806:120:R
        sysbench-3806  [02]   151.416815:   3806:120:S ==>  3798:120:R
          mysqld-3802  [05]   151.416817:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416820:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.416828:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.416829:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.416830:   3803:120:R   +  3791:120:S
          mysqld-3802  [05]   151.416832:   3802:120:R   +  3810:120:S
          mysqld-3796  [01]   151.416838:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.416839:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.416840:   3804:120:R   +  3796:120:S
          mysqld-3791  [07]   151.416842:   3791:120:R   +  3803:120:S
        sysbench-3804  [01]   151.416849:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416852:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.416865:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416867:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.416873:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.416875:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.416876:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416879:   3802:120:R   +  3810:120:S
          mysqld-3796  [01]   151.416882:   3796:120:S ==>  3804:120:R
        sysbench-3803  [07]   151.416884:   3803:120:S ==>  3791:120:R
        sysbench-3804  [01]   151.416885:   3804:120:R   +  3796:120:S
          mysqld-3791  [07]   151.416887:   3791:120:R   +  3803:120:S
        sysbench-3804  [01]   151.416893:   3804:120:S ==>  3796:120:R
          mysqld-3796  [01]   151.416896:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.416909:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416911:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.416916:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.416918:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.416920:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416923:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.416927:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.416930:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.416951:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416953:   3810:120:R   +  3802:120:S
          mysqld-3791  [07]   151.416958:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.416960:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.416961:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.416964:   3802:120:R   +  3810:120:S
        sysbench-3803  [07]   151.416969:   3803:120:S ==>  3791:120:R
          mysqld-3791  [07]   151.416972:   3791:120:R   +  3803:120:S
          mysqld-3802  [05]   151.416993:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.416996:   3810:120:R   +  3802:120:S
          mysqld-3800  [03]   151.416996:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.416998:   3800:120:R ==>  3808:120:R
          mysqld-3791  [07]   151.417002:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.417004:   3803:120:R   +  3791:120:S
        sysbench-3810  [05]   151.417004:   3810:120:S ==>  3802:120:R
          mysqld-3802  [05]   151.417007:   3802:120:R   +  3810:120:S
          mysqld-3796  [01]   151.417008:   3796:120:S ==>  3804:120:R
        sysbench-3804  [01]   151.417010:   3804:120:R   +  3796:120:S
        sysbench-3803  [07]   151.417013:   3803:120:S ==>  3791:120:R
        sysbench-3808  [03]   151.417015:   3808:120:S ==>  3800:120:R
          mysqld-3791  [07]   151.417016:   3791:120:R   +  3803:120:S
          mysqld-3800  [03]   151.417023:   3800:120:R   +  3808:120:S
          mysqld-3800  [03]   151.417024:   3800:120:R ==>  3808:120:R
        sysbench-3808  [03]   151.417025:   3808:120:S ==>  3800:120:R
        sysbench-3804  [01]   151.417026:   3804:120:S ==>  3796:120:R
          mysqld-3798  [02]   151.417027:   3798:120:R   +  3806:120:S
          mysqld-3798  [02]   151.417029:   3798:120:R ==>  3806:120:R
          mysqld-3796  [01]   151.417030:   3796:120:R   +  3804:120:S
          mysqld-3802  [05]   151.417036:   3802:120:S ==>  3810:120:R
        sysbench-3810  [05]   151.417039:   3810:120:R   +  3802:120:S
        sysbench-3806  [02]   151.417039:   3806:120:S ==>  3798:120:R
          mysqld-3791  [07]   151.417045:   3791:120:S ==>  3803:120:R
        sysbench-3810  [05]   151.417047:   3810:120:S ==>  3802:120:R
        sysbench-3803  [07]   151.417048:   3803:120:R   +  3791:120:S
#5 lore raw
Quoted 7 lines · Lin Ming

A huge trace result file, see the attachment for more.

# tracer: sched_switch
#
# TASK-PID CPU# TIMESTAMP FUNCTION
# | | | | |

          mysqld-3799  [07]   151.411009:   3799:120:R ==>  3807:120:R

...

Quoted 1 line
        sysbench-3803  [07]   151.417048:   3803:120:R   +  3791:120:S

ok, this only spans 6 milliseconds, which is probably a bit too light.

Could you please increase the trace size 32-fold (via /debug/tracing/trace_entries), and send us the (large!) trace file compressed via bzip2 -9, off-list (or via a web link)? [anyone on-list can have a look at the smaller trace file already]

Thanks,

Ingo

author
#6 lore raw
Quoted 10 lines · Ingo Molnar
Quoted 7 lines · Lin Ming

A huge trace result file, see the attachment for more.

# tracer: sched_switch
#
# TASK-PID CPU# TIMESTAMP FUNCTION
# | | | | |

          mysqld-3799  [07]   151.411009:   3799:120:R ==>  3807:120:R

...

Quoted 8 lines
Quoted 1 line
        sysbench-3803  [07]   151.417048:   3803:120:R   +  3791:120:S

ok, this only spans 6 milliseconds, which is probably a bit too light.

Could you please increase the trace size 32-fold (via /debug/tracing/trace_entries), and send us the (large!) trace file compressed via bzip2 -9, off-list (or via a web link)? [anyone on-list can have a look at the smaller trace file already]

The large trace file also available at below link,
http://myfreefilehosting.com/f/94991018dc_1.31MB

Quoted 4 lines

Thanks,

Ingo

#7 lore raw
In reply to Lin Ming
Quoted 78 lines · Lin Ming
Quoted 70 lines · Peter Zijlstra
Quoted 69 lines · Lin Ming

Comparing with 2.6.27-rc4, oltp has ~10% regression with 2.6.27-rc5 on 8-core stoakley machine.

Run oltp with 8 threads 120 seconds, vmstat shows much more idle time, about ~30%

procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------

 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
10  0      0 7822824  42240 123740    0    0   312    47  442 1613  3  2 88  6  0
 9  0      0 7822312  42240 123764    0    0     0    16 26691 232566 56 14 30  0  0
13  0      0 7821940  42240 123764    0    0     0    16 26661 228689 54 14 32  0  0
 8  0      0 7821320  42240 123764    0    0     0    16 31508 263765 61 17 23  0  0
12  0      0 7820948  42240 123764    0    0    16    16 28666 242402 57 15 28  0  0
 9  0      0 7820584  42240 123780    0    0     0    16 27107 230804 56 14 30  0  0
10  0      0 7819964  42240 123796    0    0    16   612 27599 244037 55 16 29  0  0
11  0      0 7819356  42240 123796    0    0     0    64 23540 209713 51 13 36  0  0
10  0      0 7819212  42240 123796    0    0     0    32 25674 224205 54 13 32  0  0
10  0      0 7818716  42240 123796    0    0     0    20 30106 257161 59 16 25  0  0
 7  0      0 7818468  42240 123796    0    0     0    16 28356 241551 57 14 29  0  0
10  0      0 7818096  42240 123796    0    0     0    16 39174 273656 64 16 20  0  0
12  0      0 7817724  42240 123796    0    0     0    20 39688 276936 63 16 20  0  0
11  0      0 7817352  42240 123796    0    0     0    16 42543 285192 66 16 18  0  0
 9  0      0 7817352  42240 123796    0    0     0    16 37083 259830 62 14 24  0  0
 8  0      0 7817104  42240 123796    0    0     0    16 37450 259160 61 15 23  0  0
10  0      0 7816516  42240 123796    0    0     0    64 37425 261870 61 16 23  0  0
11  0      0 7815896  42240 123812    0    0    16    16 41558 279320 66 16 18  0  0
 9  0      0 7815648  42240 123812    0    0     0    16 34017 235741 59 14 28  0  0
10  0      0 7815152  42240 123812    0    0     0    16 35642 248888 60 14 26  0  0
 9  0      0 7814532  42240 123812    0    0     0    16 38517 263220 63 15 22  0  0
 9  0      0 7814160  42240 123812    0    0     0    20 35965 246487 61 14 25  0  0
10  0      0 7814036  42240 123812    0    0     0    16 33852 236313 59 13 28  0  0
11  0      0 7813664  42240 123812    0    0     0    16 34958 244819 59 14 27  0  0
10  0      0 7813416  42240 123812    0    0     0    16 26106 202062 53 10 37  0  0
10  0      0 7812672  42240 123812    0    0     0    16 31174 222714 56 12 32  0  0
 9  0      0 7812300  42240 123812    0    0     0   276 25089 196813 52 11 38  0  0
 9  0      0 7812060  42240 123812    0    0     0    16 31877 228004 57 12 31  0  0

Bisect located below patch,
after reverted this patch the regression disappear.

commit 354879bb977e06695993435745f06a0f6d39ce2b

Author: Peter Zijlstra <[email protected]>
Date:   Mon Aug 25 17:15:34 2008 +0200

sched_clock: fix cpu_clock()

This patch fixes 3 issues:

a) it removes the dependency on jiffies, because jiffies are
incremented
by a single CPU, and the tick is not synchronized between CPUs.
Therefore
relying on it to calculate a window to clip whacky TSC values
doesn't work
as it can drift around.

So instead use [GTOD, GTOD+TICK_NSEC) as the window.

b) __update_sched_clock() did (roughly speaking):

delta = sched_clock() - scd->tick_raw;
clock += delta;

Which gives exponential growth, instead of linear.

c) allows the sched_clock_cpu() value to warp the u64 without
breaking.

Quoted 6 lines
Quoted 1 line

the results are more reliable sched_clock() deltas:

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

Quoted 7 lines

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

task migration failed because of task_hot, the system is un-balanced?

Ah, that makes sense, a more accurate clock could indeed make more tasks hot.

Can you try fiddling with: /proc/sys/kernel/sched_migration_cost ?

Also, we used to have some auto-tuning in there, which dissapeared some time ago, gregory brought it back to live recently, perhaps he likes to share? :-)

author
#8 lore raw
Quoted 12 lines
Quoted 12 lines
Quoted 5 lines

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

Quoted 6 lines
Quoted 1 line

task migration failed because of task_hot, the system is un-balanced?

Ah, that makes sense, a more accurate clock could indeed make more tasks hot.

Can you try fiddling with: /proc/sys/kernel/sched_migration_cost ?

sched_migration_cost regression
---------------------- -------------
50000 ~6%
0 ~8%
500000 (default) ~10%
5000000 ~14%
-1 ~19%

Quoted 5 lines

Also, we used to have some auto-tuning in there, which dissapeared some time ago, gregory brought it back to live recently, perhaps he likes to share? :-)

#9 lore raw
Quoted 18 lines · Lin Ming
Quoted 11 lines
Quoted 11 lines
Quoted 4 lines

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

Quoted 6 lines
Quoted 1 line

task migration failed because of task_hot, the system is un-balanced?

Ah, that makes sense, a more accurate clock could indeed make more tasks hot.

Can you try fiddling with: /proc/sys/kernel/sched_migration_cost ?

Quoted 7 lines

sched_migration_cost regression
---------------------- -------------
50000 ~6%
0 ~8%
500000 (default) ~10%
5000000 ~14%
-1 ~19%

at 50000 (~6%), is the predominant difference in schedstats still the nr_failed_migrations_hot?

author
#10 lore raw
Quoted 27 lines · Peter Zijlstra
Quoted 18 lines · Lin Ming
Quoted 11 lines
Quoted 11 lines
Quoted 4 lines

Thats bizarre... that just indicates the better clock, which should give better (read fairer) scheduling hurts your workload.

Is there anything I can run to see if we can fix the scheduler perhaps?

I observed schedstats of sysbench, there's more
"nr_failed_migrations_hot"

2.6.27-rc4: se.nr_failed_migrations_hot 11
2.6.27-rc5: se.nr_failed_migrations_hot 95

Quoted 6 lines
Quoted 1 line

task migration failed because of task_hot, the system is un-balanced?

Ah, that makes sense, a more accurate clock could indeed make more tasks hot.

Can you try fiddling with: /proc/sys/kernel/sched_migration_cost ?

Quoted 7 lines

sched_migration_cost regression
---------------------- -------------
50000 ~6%
0 ~8%
500000 (default) ~10%
5000000 ~14%
-1 ~19%

Quoted 2 lines

at 50000 (~6%), is the predominant difference in schedstats still the nr_failed_migrations_hot?

Yes, it's strange that nr_failed_migrations_hot (cost=50000) is larger than (cost=500000)

sched_migration_cost = 50000
----------------------------------------------------------

se.exec_start                      :       3475158.689868
se.vruntime                        :        935215.526989
se.sum_exec_runtime                :         14358.431942
se.avg_overlap                     :             0.000000
se.wait_start                      :             0.000000
se.sleep_start                     :       3475158.689868
se.block_start                     :             0.000000
se.sleep_max                       :            37.910779
se.block_max                       :             0.033796
se.exec_max                        :             0.121028
se.slice_max                       :             0.000000
se.wait_max                        :             6.007209
se.wait_sum                        :         25029.649588
se.wait_count                      :              1954948
sched_info.bkl_count               :                    0
se.nr_migrations                   :                 2521
se.nr_migrations_cold              :                    0
se.nr_failed_migrations_affine     :                    0
se.nr_failed_migrations_running    :                  147
se.nr_failed_migrations_hot        :                  123
se.nr_forced_migrations            :                    0
se.nr_forced2_migrations           :                 1236
se.nr_wakeups                      :              1952827
se.nr_wakeups_sync                 :              1944785
se.nr_wakeups_migrate              :                 2479
se.nr_wakeups_local                :              1930122
se.nr_wakeups_remote               :                22705
se.nr_wakeups_affine               :                13252
se.nr_wakeups_affine_attempts      :               825950
se.nr_wakeups_passive              :                    0
se.nr_wakeups_idle                 :                    0
avg_atom                           :             0.007355
avg_per_cpu                        :             5.695530
nr_switches                        :              1951994
nr_voluntary_switches              :              1949963
nr_involuntary_switches            :                 2031
se.load.weight                     :                 1024
policy                             :                    0
prio                               :                  120
clock-delta                        :                   81

sched_migration_cost = 500000
----------------------------------------------------------

se.exec_start                      :       3719706.806593
se.vruntime                        :       1049301.798712
se.sum_exec_runtime                :         14636.764384
se.avg_overlap                     :             0.000121
se.wait_start                      :             0.000000
se.sleep_start                     :       3719706.806593
se.block_start                     :             0.000000
se.sleep_max                       :           214.989106
se.block_max                       :             0.036240
se.exec_max                        :             0.147829
se.slice_max                       :             0.000000
se.wait_max                        :             6.475864
se.wait_sum                        :         24932.530233
se.wait_count                      :              1922627
sched_info.bkl_count               :                    0
se.nr_migrations                   :                 2314
se.nr_migrations_cold              :                    0
se.nr_failed_migrations_affine     :                    0
se.nr_failed_migrations_running    :                   71
se.nr_failed_migrations_hot        :                   76
se.nr_forced_migrations            :                    0
se.nr_forced2_migrations           :                 1735
se.nr_wakeups                      :              1920337
se.nr_wakeups_sync                 :              1908767
se.nr_wakeups_migrate              :                 2272
se.nr_wakeups_local                :              1898194
se.nr_wakeups_remote               :                22143
se.nr_wakeups_affine               :                 1218
se.nr_wakeups_affine_attempts      :               820118
se.nr_wakeups_passive              :                    0
se.nr_wakeups_idle                 :                    0
avg_atom                           :             0.007624
avg_per_cpu                        :             6.325308
nr_switches                        :              1919596
nr_voluntary_switches              :              1917396
nr_involuntary_switches            :                 2200
se.load.weight                     :                 1024
policy                             :                    0
prio                               :                  120
clock-delta                        :                   69
#11 lore raw
In reply to Peter Zijlstra

Hi Peter,

Quoted 6 lines · Peter Zijlstra

Also, we used to have some auto-tuning in there, which dissapeared some time ago, gregory brought it back to live recently, perhaps he likes to share? :-)

Per your request, I brought those "revert" patches I had in the PQ branch forward to 2.6.27-rc5. They are completely unbuilt and untested, but here for your experimentation.

-Greg

---

Gregory Haskins (4):
sched: make task_hot() once again use sd->cache_hot_time Revert "sched: zap the migration init / cache-hot balancing code" Revert "[PATCH] sched: remove cache_hot_time"
revert "sched: sched_cacheflush is now unused"

 Documentation/kernel-parameters.txt |   43 +++
 arch/ia64/kernel/setup.c            |   15 +
 arch/mips/kernel/smp.c              |   11 +
 arch/sparc/kernel/smp.c             |   10 +
 arch/sparc64/kernel/smp.c           |   27 ++
 arch/x86/kernel/smpboot.c           |   12 +
 include/asm-m32r/system.h           |   10 +
 include/asm-mips/system.h           |   10 +
 include/asm-parisc/system.h         |   11 +
 include/asm-x86/system.h            |    9 +
 include/linux/sched.h               |    7 
 include/linux/topology.h            |    1 
 kernel/sched.c                      |  499 ++++++++++++++++++++++++++++++++++-
 13 files changed, 656 insertions(+), 9 deletions(-)
Signature · 1 line
Signature
#12 lore raw

PATCH 1/4 revert "sched: sched_cacheflush is now unused"

revert c41917df8a1adde34864116ce2231a7fe308d2ff

Author: Ralf Baechle <[email protected]>
Date: Thu Jul 19 21:28:35 2007 +0200

[PATCH] sched: sched_cacheflush is now unused

Since Ingo's recent scheduler rewrite which was merged as commit 0437e109e1841607f2988891eaa36c531c6aa6ac sched_cacheflush is unused.

We want to restore sd->cache_hot_time for better migration decisions.

---

 arch/ia64/kernel/setup.c    |    9 +++++++++
 include/asm-m32r/system.h   |   10 ++++++++++
 include/asm-mips/system.h   |   10 ++++++++++
 include/asm-parisc/system.h |   11 +++++++++++
 include/asm-x86/system.h    |    9 +++++++++
 5 files changed, 49 insertions(+), 0 deletions(-)
arch/ia64/kernel/setup.c +9 -0
index c27d5b2..c0050ab 100644
@@ -1051,6 +1051,15 @@ cpu_init (void)
 	pm_idle = default_idle;
 }
 
+/*
+ * On SMP systems, when the scheduler does migration-cost autodetection,
+ * it needs a way to flush as much of the CPU's caches as possible.
+ */
+void sched_cacheflush(void)
+{
+	ia64_sal_cache_flush(3);
+}
+
 void __init
 check_bugs (void)
 {
include/asm-m32r/system.h +10 -0
index 70a57c8..d2f9559 100644
@@ -54,6 +54,16 @@
 	); \
 } while(0)
 
+/*
+ * On SMP systems, when the scheduler does migration-cost autodetection,
+ * it needs a way to flush as much of the CPU's caches as possible.
+ *
+ * TODO: fill this in!
+ */
+static inline void sched_cacheflush(void)
+{
+}
+
 /* Interrupt Control */
 #if !defined(CONFIG_CHIP_M32102) && !defined(CONFIG_CHIP_M32104)
 #define local_irq_enable() \
include/asm-mips/system.h +10 -0
index a944eda..aaba821 100644
@@ -78,6 +78,16 @@ do {									\
 		write_c0_userlocal(current_thread_info()->tp_value);	\
 } while (0)
 
+/*
+ * On SMP systems, when the scheduler does migration-cost autodetection,
+ * it needs a way to flush as much of the CPU's caches as possible.
+ *
+ * TODO: fill this in!
+ */
+static inline void sched_cacheflush(void)
+{
+}
+
 static inline unsigned long __xchg_u32(volatile int * m, unsigned int val)
 {
 	__u32 retval;
include/asm-parisc/system.h +11 -0
index ee80c92..21fbfc5 100644
@@ -48,6 +48,17 @@ extern struct task_struct *_switch_to(struct task_struct *, struct task_struct *
 	(last) = _switch_to(prev, next);			\
 } while(0)
 
+/*
+ * On SMP systems, when the scheduler does migration-cost autodetection,
+ * it needs a way to flush as much of the CPU's caches as possible.
+ *
+ * TODO: fill this in!
+ */
+static inline void sched_cacheflush(void)
+{
+}
+
+
 /* interrupt control */
 #define local_save_flags(x)	__asm__ __volatile__("ssm 0, %0" : "=r" (x) : : "memory")
 #define local_irq_disable()	__asm__ __volatile__("rsm %0,%%r0\n" : : "i" (PSW_I) : "memory" )
include/asm-x86/system.h +9 -0
index 983ce37..7847702 100644
@@ -306,6 +306,15 @@ void enable_hlt(void);
 
 void cpu_idle_wait(void);
 
+/*
+ * On SMP systems, when the scheduler does migration-cost autodetection,
+ * it needs a way to flush as much of the CPU's caches as possible:
+ */
+static inline void sched_cacheflush(void)
+{
+	wbinvd();
+}
+
 extern unsigned long arch_align_stack(unsigned long sp);
 extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
 
#13 lore raw

PATCH 2/4 Revert "[PATCH] sched: remove cache_hot_time"

In reply to Gregory Haskins

This reverts commit 362a7016637648c6aefc98b706298baedfaa1543.

---

 include/linux/sched.h    |    1 +
 include/linux/topology.h |    1 +
 kernel/sched.c           |    8 +++++---
 3 files changed, 7 insertions(+), 3 deletions(-)
include/linux/sched.h +1 -0
index cfb0d87..5619f3c 100644
@@ -777,6 +777,7 @@ struct sched_domain {
 	unsigned long max_interval;	/* Maximum balance interval ms */
 	unsigned int busy_factor;	/* less balancing by factor if busy */
 	unsigned int imbalance_pct;	/* No balance until over watermark */
+	unsigned long long cache_hot_time; /* Task considered cache hot (ns) */
 	unsigned int cache_nice_tries;	/* Leave cache hot tasks for # tries */
 	unsigned int busy_idx;
 	unsigned int idle_idx;
include/linux/topology.h +1 -0
index 2158fc0..f6a7928 100644
@@ -162,6 +162,7 @@ void arch_update_cpu_topology(void);
 	.max_interval		= 64*num_online_cpus(),	\
 	.busy_factor		= 128,			\
 	.imbalance_pct		= 133,			\
+	.cache_hot_time		= (10*1000000),		\
 	.cache_nice_tries	= 1,			\
 	.busy_idx		= 3,			\
 	.idle_idx		= 3,			\
kernel/sched.c +5 -3
index 9a1ddb8..0ca5218 100644
@@ -6223,7 +6223,7 @@ set_table_entry(struct ctl_table *entry,
 static struct ctl_table *
 sd_alloc_ctl_domain_table(struct sched_domain *sd)
 {
-	struct ctl_table *table = sd_alloc_ctl_entry(12);
+	struct ctl_table *table = sd_alloc_ctl_entry(13);
 
 	if (table == NULL)
 		return NULL;
@@ -6249,9 +6249,11 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
 	set_table_entry(&table[9], "cache_nice_tries",
 		&sd->cache_nice_tries,
 		sizeof(int), 0644, proc_dointvec_minmax);
-	set_table_entry(&table[10], "flags", &sd->flags,
+	set_table_entry(&table[10], "cache_hot_time", &sd->cache_hot_time,
+		sizeof(long long), 0644, proc_doulongvec_minmax);
+	set_table_entry(&table[11], "flags", &sd->flags,
 		sizeof(int), 0644, proc_dointvec_minmax);
-	/* &table[11] is terminator */
+	/* &table[12] is terminator */
 
 	return table;
 }
#14 lore raw

PATCH 3/4 Revert "sched: zap the migration init / cache-hot balancing code"

In reply to Gregory Haskins
Quoted 1 line

From commit: 0437e109e1841607f2988891eaa36c531c6aa6ac

We want to restore the concept of using empirical cache_hot data for managing migrations.

---

 Documentation/kernel-parameters.txt |   43 +++
 arch/ia64/kernel/setup.c            |    6 
 arch/mips/kernel/smp.c              |   11 +
 arch/sparc/kernel/smp.c             |   10 +
 arch/sparc64/kernel/smp.c           |   27 ++
 arch/x86/kernel/smpboot.c           |   12 +
 include/linux/sched.h               |    6 
 kernel/sched.c                      |  483 +++++++++++++++++++++++++++++++++++
 8 files changed, 598 insertions(+), 0 deletions(-)
Documentation/kernel-parameters.txt +43 -0
index 1150444..6e7b78f 100644
@@ -1247,6 +1247,49 @@ and is between 256 and 4096 characters. It is defined in the file
 
 	mga=		[HW,DRM]
 
+	migration_cost=
+			[KNL,SMP] debug: override scheduler migration costs
+			Format: <level-1-usecs>,<level-2-usecs>,...
+			This debugging option can be used to override the
+			default scheduler migration cost matrix. The numbers
+			are indexed by 'CPU domain distance'.
+			E.g. migration_cost=1000,2000,3000 on an SMT NUMA
+			box will set up an intra-core migration cost of
+			1 msec, an inter-core migration cost of 2 msecs,
+			and an inter-node migration cost of 3 msecs.
+
+			WARNING: using the wrong values here can break
+			scheduler performance, so it's only for scheduler
+			development purposes, not production environments.
+
+	migration_debug=
+			[KNL,SMP] migration cost auto-detect verbosity
+			Format=<0|1|2>
+			If a system's migration matrix reported at bootup
+			seems erroneous then this option can be used to
+			increase verbosity of the detection process.
+			We default to 0 (no extra messages), 1 will print
+			some more information, and 2 will be really
+			verbose (probably only useful if you also have a
+			serial console attached to the system).
+
+	migration_factor=
+			[KNL,SMP] multiply/divide migration costs by a factor
+			Format=<percent>
+			This debug option can be used to proportionally
+			increase or decrease the auto-detected migration
+			costs for all entries of the migration matrix.
+			E.g. migration_factor=150 will increase migration
+			costs by 50%. (and thus the scheduler will be less
+			eager migrating cache-hot tasks)
+			migration_factor=80 will decrease migration costs
+			by 20%. (thus the scheduler will be more eager to
+			migrate tasks)
+
+			WARNING: using the wrong values here can break
+			scheduler performance, so it's only for scheduler
+			development purposes, not production environments.
+
 	mminit_loglevel=
 			[KNL] When CONFIG_DEBUG_MEMORY_INIT is set, this
 			parameter allows control of the logging verbosity for
arch/ia64/kernel/setup.c +6 -0
index c0050ab..d2e1724 100644
@@ -854,6 +854,7 @@ static void __cpuinit
 get_max_cacheline_size (void)
 {
 	unsigned long line_size, max = 1;
+	unsigned int cache_size = 0;
 	u64 l, levels, unique_caches;
         pal_cache_config_info_t cci;
         s64 status;
@@ -883,6 +884,8 @@ get_max_cacheline_size (void)
 		line_size = 1 << cci.pcci_line_size;
 		if (line_size > max)
 			max = line_size;
+		if (cache_size < cci.pcci_cache_size)
+			cache_size = cci.pcci_cache_size;
 		if (!cci.pcci_unified) {
 			status = ia64_pal_cache_config_info(l,
 						    /* cache_type (instruction)= */ 1,
@@ -899,6 +902,9 @@ get_max_cacheline_size (void)
 			ia64_i_cache_stride_shift = cci.pcci_stride;
 	}
   out:
+#ifdef CONFIG_SMP
+	max_cache_size = max(max_cache_size, cache_size);
+#endif
 	if (max > ia64_max_cacheline_size)
 		ia64_max_cacheline_size = max;
 }
arch/mips/kernel/smp.c +11 -0
index 4410f17..cb63c56 100644
@@ -53,6 +53,16 @@ int __cpu_logical_map[NR_CPUS];		/* Map logical to physical */
 EXPORT_SYMBOL(phys_cpu_present_map);
 EXPORT_SYMBOL(cpu_online_map);
 
+/* This happens early in bootup, can't really do it better */
+static void smp_tune_scheduling (void)
+{
+ 	struct cache_desc *cd = &current_cpu_data.scache;
+ 	unsigned long cachesize = cd->linesz * cd->sets * cd->ways;
+	
+ 	if (cachesize > max_cache_size)
+ 		max_cache_size = cachesize;
+}
+ 
 extern void cpu_idle(void);
 
 /* Number of TCs (or siblings in Intel speak) per CPU core */
@@ -181,6 +191,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 {
 	init_new_context(current, &init_mm);
 	current_thread_info()->cpu = 0;
+	smp_tune_scheduling();
 	mp_ops->prepare_cpus(max_cpus);
 	set_cpu_sibling_map(0);
 #ifndef CONFIG_HOTPLUG_CPU
arch/sparc/kernel/smp.c +10 -0
index 1619ec1..44a0448 100644
@@ -63,6 +63,16 @@ void __cpuinit smp_store_cpu_info(int id)
 	cpu_data(id).prom_node = cpu_node;
 	cpu_data(id).mid = cpu_get_hwmid(cpu_node);
 
+	/* this is required to tune the scheduler correctly */
+	/* is it possible to have CPUs with different cache sizes? */
+	if (id == boot_cpu_id) {
+		int cache_line,cache_nlines;
+		cache_line = 0x20;
+		cache_line = prom_getintdefault(cpu_node, "ecache-line-size", cache_line);
+		cache_nlines = 0x8000;
+		cache_nlines = prom_getintdefault(cpu_node, "ecache-nlines", cache_nlines);
+		max_cache_size = cache_line * cache_nlines;
+	}
 	if (cpu_data(id).mid < 0)
 		panic("No MID found for CPU%d at node 0x%08d", id, cpu_node);
 }
arch/sparc64/kernel/smp.c +27 -0
index 743ccad..926072b 100644
@@ -1174,8 +1174,35 @@ int setup_profiling_timer(unsigned int multiplier)
 	return -EINVAL;
 }
 
+static void __init smp_tune_scheduling(void)
+{
+	unsigned int smallest = ~0U;
+	int i;
+
+	for (i = 0; i < NR_CPUS; i++) {
+		unsigned int val = cpu_data(i).ecache_size;
+
+		if (val && val < smallest)
+			smallest = val;
+	}
+
+	/* Any value less than 256K is nonsense.  */
+	if (smallest < (256U * 1024U))
+		smallest = 256 * 1024;
+
+	max_cache_size = smallest;
+
+	if (smallest < 1U * 1024U * 1024U)
+		printk(KERN_INFO "Using max_cache_size of %uKB\n",
+		       smallest / 1024U);
+	else
+		printk(KERN_INFO "Using max_cache_size of %uMB\n",
+		       smallest / 1024U / 1024U);
+}
+
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
+	smp_tune_scheduling();
 }
 
 void __devinit smp_prepare_boot_cpu(void)
arch/x86/kernel/smpboot.c +12 -0
index 7985c5b..c98fdc5 100644
@@ -1029,6 +1029,17 @@ int __cpuinit native_cpu_up(unsigned int cpu)
 	return 0;
 }
 
+static void smp_tune_scheduling(void)
+{
+	if (cpu_khz) {
+		/* cache size in kB */
+		long cachesize = boot_cpu_data.x86_cache_size;
+
+		if (cachesize > 0)
+			max_cache_size = cachesize * 1024;
+	}
+}
+
 /*
  * Fall back to non SMP mode after errors.
  *
@@ -1177,6 +1188,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	smp_store_cpu_info(0); /* Final full version of the data */
 	boot_cpu_logical_apicid = logical_smp_processor_id();
 	current_thread_info()->cpu = 0;  /* needed? */
+	smp_tune_scheduling();
 	set_cpu_sibling_map(0);
 
 	if (smp_sanity_check(max_cpus) < 0) {
include/linux/sched.h +6 -0
index 5619f3c..5046e3a 100644
@@ -831,6 +831,12 @@ extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
 				    struct sched_domain_attr *dattr_new);
 extern int arch_reinit_sched_domains(void);
 
+/*
+ * Maximum cache size the migration-costs auto-tuning code will
+ * search from:
+ */
+extern unsigned int max_cache_size;
+
 #else /* CONFIG_SMP */
 
 struct sched_domain_attr;
kernel/sched.c +483 -0

495 lines not on this page · raw message

#15 lore raw

PATCH 4/4 sched: make task_hot() once again use sd->cache_hot_time

In reply to Gregory Haskins

---

 kernel/sched.c |    8 ++------
 1 files changed, 2 insertions(+), 6 deletions(-)
kernel/sched.c +2 -6
index fd28b64..b5f8640 100644
@@ -1778,6 +1778,7 @@ static int
 task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 {
 	s64 delta;
+	s64 migration_cost = (s64)(sd ? sd->cache_hot_time : sysctl_sched_migration_cost);
 
 	/*
 	 * Buddy candidates are cache hot:
@@ -1788,14 +1789,9 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 	if (p->sched_class != &fair_sched_class)
 		return 0;
 
-	if (sysctl_sched_migration_cost == -1)
-		return 1;
-	if (sysctl_sched_migration_cost == 0)
-		return 0;
-
 	delta = now - p->se.exec_start;
 
-	return delta < (s64)sysctl_sched_migration_cost;
+	return delta < migration_cost;
 }
 
 
author
#16 lore raw
In reply to Peter Zijlstra

It does not help after these 4 patches applied.

Gregory Haskins (4):
sched: make task_hot() once again use sd->cache_hot_time Revert "sched: zap the migration init / cache-hot balancing code" Revert "[PATCH] sched: remove cache_hot_time"
revert "sched: sched_cacheflush is now unused"

#17 lore raw
In reply to Lin Ming
Quoted 20 lines

------- Comment #4 from [email protected] 2008-09-17 17:55 ------- sched_switch trace of oltp in 2.6.27-rc4
http://myfreefilehosting.com/f/fc6c8eaacf_1.31MB

sched_switch trace of oltp in 2.6.27-rc5
http://myfreefilehosting.com/f/a2f9aea1b0_0.42MB

Compared the 2 trace files, you can find that (at most time), with 2.6.27-rc5, mysql switches to sysbench when it's going to sleep

          mysqld-3791  [07]   151.421836:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.421876:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.421878:   3803:120:R   +  3791:120:S
        sysbench-3803  [07]   151.421887:   3803:120:S ==>  3791:120:R

with 2.6.27-rc4, mysql switches to sysbench when it's still running

          mysqld-3674  [07]    95.960220:   3674:120:R   +  3687:120:S
          mysqld-3674  [07]    95.960220:   3674:120:R ==>  3687:120:R
        sysbench-3687  [07]    95.960220:   3687:120:S ==>  3674:120:R
Quoted 5 lines

So with 2.6.27-rc5, sysbench wakes up mysql and then switches to it (mysql is in sleep state)

With 2.6.27-rc4, sysbench switches to mysql, no need to wake up it (mysql is still in running state)

Ming, how does this work for you?

---
Subject: sched: wakeup preempt when small overlap

Aggresively preempt a task if its avg overlap is very small, this should avoid the task going to sleep and find it still running when we schedule back to it - saving a wakeup.

---

include/linux/sched.h +1 -1
index 7b4592c..cb44774 100644
@@ -897,7 +897,7 @@ struct sched_class {
 	void (*yield_task) (struct rq *rq);
 	int  (*select_task_rq)(struct task_struct *p, int sync);
 
-	void (*check_preempt_curr) (struct rq *rq, struct task_struct *p);
+	void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int sync);
 
 	struct task_struct * (*pick_next_task) (struct rq *rq);
 	void (*put_prev_task) (struct rq *rq, struct task_struct *p);
kernel/sched.c +6 -6
index 32d56d6..e17d506 100644
@@ -609,9 +609,9 @@ struct rq {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
-static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
+static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
 {
-	rq->curr->sched_class->check_preempt_curr(rq, p);
+	rq->curr->sched_class->check_preempt_curr(rq, p, sync);
 }
 
 static inline int cpu_of(struct rq *rq)
@@ -2299,7 +2299,7 @@ out_activate:
 
 out_running:
 	trace_sched_wakeup(rq, p);
-	check_preempt_curr(rq, p);
+	check_preempt_curr(rq, p, sync);
 
 	p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
@@ -2432,7 +2432,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
 		inc_nr_running(rq);
 	}
 	trace_sched_wakeup_new(rq, p);
-	check_preempt_curr(rq, p);
+	check_preempt_curr(rq, p, 0);
 #ifdef CONFIG_SMP
 	if (p->sched_class->task_wake_up)
 		p->sched_class->task_wake_up(rq, p);
@@ -2889,7 +2889,7 @@ static void pull_task(struct rq *src_rq, struct task_struct *p,
 	 * Note that idle threads have a prio of MAX_PRIO, for this test
 	 * to be always true for them.
 	 */
-	check_preempt_curr(this_rq, p);
+	check_preempt_curr(this_rq, p, 0);
 }
 
 /*
@@ -6015,7 +6015,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
 	set_task_cpu(p, dest_cpu);
 	if (on_rq) {
 		activate_task(rq_dest, p, 0);
-		check_preempt_curr(rq_dest, p);
+		check_preempt_curr(rq_dest, p, 0);
 	}
 done:
 	ret = 1;
kernel/sched_fair.c +12 -5
index a10ac0b..82a907c 100644
@@ -1331,7 +1331,7 @@ static inline int depth_se(struct sched_entity *se)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
+static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
 {
 	struct task_struct *curr = rq->curr;
 	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
@@ -1348,6 +1348,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
 	if (unlikely(se == pse))
 		return;
 
+	cfs_rq_of(pse)->next = pse;
+
 	/*
 	 * We can come here with TIF_NEED_RESCHED already set from new task
 	 * wake up path.
@@ -1355,8 +1357,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
 	if (test_tsk_need_resched(curr))
 		return;
 
-	cfs_rq_of(pse)->next = pse;
-
 	/*
 	 * Batch tasks do not preempt (their preemption is driven by
 	 * the tick):
@@ -1367,6 +1367,13 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
 	if (!sched_feat(WAKEUP_PREEMPT))
 		return;
 
+	if (sched_feat(WAKEUP_OVERLAP) && sync && 
+			se->avg_overlap < sysctl_sched_migration_cost &&
+			pse->avg_overlap < sysctl_sched_migration_cost) {
+		resched_task(curr);
+		return;
+	}
+
 	/*
 	 * preemption test can be made between sibling entities who are in the
 	 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
@@ -1649,7 +1656,7 @@ static void prio_changed_fair(struct rq *rq, struct task_struct *p,
 		if (p->prio > oldprio)
 			resched_task(rq->curr);
 	} else
-		check_preempt_curr(rq, p);
+		check_preempt_curr(rq, p, 0);
 }
 
 /*
@@ -1666,7 +1673,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p,
 	if (running)
 		resched_task(rq->curr);
 	else
-		check_preempt_curr(rq, p);
+		check_preempt_curr(rq, p, 0);
 }
 
 /* Account for a task changing its policy or group.
kernel/sched_features.h +1 -0
index 9353ca7..bf027a7 100644
@@ -11,3 +11,4 @@ SCHED_FEAT(ASYM_GRAN, 1)
 SCHED_FEAT(LB_BIAS, 1)
 SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
 SCHED_FEAT(ASYM_EFF_LOAD, 1)
+SCHED_FEAT(WAKEUP_OVERLAP, 1)
kernel/sched_idletask.c +3 -3
index 3a4f92d..dec4cca 100644
@@ -14,7 +14,7 @@ static int select_task_rq_idle(struct task_struct *p, int sync)
 /*
  * Idle tasks are unconditionally rescheduled:
  */
-static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p)
+static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int sync)
 {
 	resched_task(rq->idle);
 }
@@ -76,7 +76,7 @@ static void switched_to_idle(struct rq *rq, struct task_struct *p,
 	if (running)
 		resched_task(rq->curr);
 	else
-		check_preempt_curr(rq, p);
+		check_preempt_curr(rq, p, 0);
 }
 
 static void prio_changed_idle(struct rq *rq, struct task_struct *p,
@@ -93,7 +93,7 @@ static void prio_changed_idle(struct rq *rq, struct task_struct *p,
 		if (p->prio > oldprio)
 			resched_task(rq->curr);
 	} else
-		check_preempt_curr(rq, p);
+		check_preempt_curr(rq, p, 0);
 }
 
 /*
kernel/sched_rt.c +1 -1
index 04875ef..2e228bd 100644
@@ -787,7 +787,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
+static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int sync)
 {
 	if (p->prio < rq->curr->prio) {
 		resched_task(rq->curr);
author
#18 lore raw

This patch restores ~5% performance.

kernel peformance
-----------------------------------

27-rc4                  100%
27-rc5                  89.6%
27-rc5+peter's patch    94.5%

Thanks,
Lin Ming

Quoted 220 lines · Peter Zijlstra
Quoted 20 lines

------- Comment #4 from [email protected] 2008-09-17 17:55 ------- sched_switch trace of oltp in 2.6.27-rc4
http://myfreefilehosting.com/f/fc6c8eaacf_1.31MB

sched_switch trace of oltp in 2.6.27-rc5
http://myfreefilehosting.com/f/a2f9aea1b0_0.42MB

Compared the 2 trace files, you can find that (at most time), with 2.6.27-rc5, mysql switches to sysbench when it's going to sleep

          mysqld-3791  [07]   151.421836:   3791:120:R   +  3803:120:S
          mysqld-3791  [07]   151.421876:   3791:120:S ==>  3803:120:R
        sysbench-3803  [07]   151.421878:   3803:120:R   +  3791:120:S
        sysbench-3803  [07]   151.421887:   3803:120:S ==>  3791:120:R

with 2.6.27-rc4, mysql switches to sysbench when it's still running

          mysqld-3674  [07]    95.960220:   3674:120:R   +  3687:120:S
          mysqld-3674  [07]    95.960220:   3674:120:R ==>  3687:120:R
        sysbench-3687  [07]    95.960220:   3687:120:S ==>  3674:120:R
Quoted 5 lines

So with 2.6.27-rc5, sysbench wakes up mysql and then switches to it (mysql is in sleep state)

With 2.6.27-rc4, sysbench switches to mysql, no need to wake up it (mysql is still in running state)

Ming, how does this work for you?

---
Subject: sched: wakeup preempt when small overlap

Aggresively preempt a task if its avg overlap is very small, this should avoid the task going to sleep and find it still running when we schedule back to it - saving a wakeup.

---

include/linux/sched.h +1 -1
index 7b4592c..cb44774 100644
@@ -897,7 +897,7 @@ struct sched_class {
        void (*yield_task) (struct rq *rq);
        int  (*select_task_rq)(struct task_struct *p, int sync);
 
-       void (*check_preempt_curr) (struct rq *rq, struct task_struct *p);
+       void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int sync);
 
        struct task_struct * (*pick_next_task) (struct rq *rq);
        void (*put_prev_task) (struct rq *rq, struct task_struct *p);
kernel/sched.c +6 -6
index 32d56d6..e17d506 100644
@@ -609,9 +609,9 @@ struct rq {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
 
-static inline void check_preempt_curr(struct rq *rq, struct task_struct *p)
+static inline void check_preempt_curr(struct rq *rq, struct task_struct *p, int sync)
 {
-       rq->curr->sched_class->check_preempt_curr(rq, p);
+       rq->curr->sched_class->check_preempt_curr(rq, p, sync);
 }
 
 static inline int cpu_of(struct rq *rq)
@@ -2299,7 +2299,7 @@ out_activate:
 
 out_running:
        trace_sched_wakeup(rq, p);
-       check_preempt_curr(rq, p);
+       check_preempt_curr(rq, p, sync);
 
        p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
@@ -2432,7 +2432,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                inc_nr_running(rq);
        }
        trace_sched_wakeup_new(rq, p);
-       check_preempt_curr(rq, p);
+       check_preempt_curr(rq, p, 0);
 #ifdef CONFIG_SMP
        if (p->sched_class->task_wake_up)
                p->sched_class->task_wake_up(rq, p);
@@ -2889,7 +2889,7 @@ static void pull_task(struct rq *src_rq, struct task_struct *p,
         * Note that idle threads have a prio of MAX_PRIO, for this test
         * to be always true for them.
         */
-       check_preempt_curr(this_rq, p);
+       check_preempt_curr(this_rq, p, 0);
 }
 
 /*
@@ -6015,7 +6015,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
        set_task_cpu(p, dest_cpu);
        if (on_rq) {
                activate_task(rq_dest, p, 0);
-               check_preempt_curr(rq_dest, p);
+               check_preempt_curr(rq_dest, p, 0);
        }
 done:
        ret = 1;
kernel/sched_fair.c +12 -5
index a10ac0b..82a907c 100644
@@ -1331,7 +1331,7 @@ static inline int depth_se(struct sched_entity *se)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
+static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
 {
        struct task_struct *curr = rq->curr;
        struct cfs_rq *cfs_rq = task_cfs_rq(curr);
@@ -1348,6 +1348,8 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
        if (unlikely(se == pse))
                return;
 
+       cfs_rq_of(pse)->next = pse;
+
        /*
         * We can come here with TIF_NEED_RESCHED already set from new task
         * wake up path.
@@ -1355,8 +1357,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
        if (test_tsk_need_resched(curr))
                return;
 
-       cfs_rq_of(pse)->next = pse;
-
        /*
         * Batch tasks do not preempt (their preemption is driven by
         * the tick):
@@ -1367,6 +1367,13 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
        if (!sched_feat(WAKEUP_PREEMPT))
                return;
 
+       if (sched_feat(WAKEUP_OVERLAP) && sync &&
+                       se->avg_overlap < sysctl_sched_migration_cost &&
+                       pse->avg_overlap < sysctl_sched_migration_cost) {
+               resched_task(curr);
+               return;
+       }
+
        /*
         * preemption test can be made between sibling entities who are in the
         * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
@@ -1649,7 +1656,7 @@ static void prio_changed_fair(struct rq *rq, struct task_struct *p,
                if (p->prio > oldprio)
                        resched_task(rq->curr);
        } else
-               check_preempt_curr(rq, p);
+               check_preempt_curr(rq, p, 0);
 }
 
 /*
@@ -1666,7 +1673,7 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p,
        if (running)
                resched_task(rq->curr);
        else
-               check_preempt_curr(rq, p);
+               check_preempt_curr(rq, p, 0);
 }
 
 /* Account for a task changing its policy or group.
kernel/sched_features.h +1 -0
index 9353ca7..bf027a7 100644
@@ -11,3 +11,4 @@ SCHED_FEAT(ASYM_GRAN, 1)
 SCHED_FEAT(LB_BIAS, 1)
 SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
 SCHED_FEAT(ASYM_EFF_LOAD, 1)
+SCHED_FEAT(WAKEUP_OVERLAP, 1)
kernel/sched_idletask.c +3 -3
index 3a4f92d..dec4cca 100644
@@ -14,7 +14,7 @@ static int select_task_rq_idle(struct task_struct *p, int sync)
 /*
  * Idle tasks are unconditionally rescheduled:
  */
-static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p)
+static void check_preempt_curr_idle(struct rq *rq, struct task_struct *p, int sync)
 {
        resched_task(rq->idle);
 }
@@ -76,7 +76,7 @@ static void switched_to_idle(struct rq *rq, struct task_struct *p,
        if (running)
                resched_task(rq->curr);
        else
-               check_preempt_curr(rq, p);
+               check_preempt_curr(rq, p, 0);
 }
 
 static void prio_changed_idle(struct rq *rq, struct task_struct *p,
@@ -93,7 +93,7 @@ static void prio_changed_idle(struct rq *rq, struct task_struct *p,
                if (p->prio > oldprio)
                        resched_task(rq->curr);
        } else
-               check_preempt_curr(rq, p);
+               check_preempt_curr(rq, p, 0);
 }
 
 /*
kernel/sched_rt.c +1 -1
index 04875ef..2e228bd 100644
@@ -787,7 +787,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
 /*
  * Preempt the current task with a newly woken task if needed:
  */
-static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
+static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int sync)
 {
        if (p->prio < rq->curr->prio) {
                resched_task(rq->curr);