sched: hang in migrate_swap
Subsystems (2)
Hi all,
While fuzzing with trinity inside a KVM tools guest, running latest -next kernel, I see to hit the following hang quite often.
The initial spew is:
[ 293.110057] BUG: soft lockup - CPU#8 stuck for 22s! [migration/8:258] [ 293.110057] Modules linked in: [ 293.110057] irq event stamp: 20828 [ 293.110057] hardirqs last enabled at (20827): [<ffffffff84385437>] restore_args+0x0/0x30 [ 293.110057] hardirqs last disabled at (20828): [<ffffffff8438f02d>] apic_timer_interrupt+0x6d/0x80 [ 293.110057] softirqs last enabled at (20826): [<ffffffff811418e7>] __do_softirq+0x447/0x4f0 [ 293.110057] softirqs last disabled at (20821): [<ffffffff81141fa3>] irq_exit+0x83/0x160 [ 293.110057] CPU: 8 PID: 258 Comm: migration/8 Tainted: G W 3.14.0-rc3-next-20140218-sasha-00008-g16140ff-dirty #109 [ 293.110057] task: ffff8805b8283000 ti: ffff8805b82c0000 task.ti: ffff8805b82c0000 [ 293.110057] RIP: 0010:[<ffffffff81203892>] [<ffffffff81203892>] multi_cpu_stop+0xc2/0x1c0 [ 293.110057] RSP: 0000:ffff8805b82c1cd8 EFLAGS: 00000293 [ 293.110057] RAX: 0000000000000001 RBX: ffffffff84385437 RCX: 0000000000000000 [ 293.110057] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8800c40bb978 [ 293.110057] RBP: ffff8805b82c1d18 R08: 0000000000000000 R09: 0000000000000000 [ 293.110057] R10: 0000000000000001 R11: 0000000000000001 R12: ffff8805b82c1c48 [ 293.110057] R13: ffff8805b8283000 R14: ffff8805b82c0000 R15: ffff8805b8283000 [ 293.110057] FS: 0000000000000000(0000) GS:ffff8805b9000000(0000) knlGS:0000000000000000 [ 293.110057] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 293.110057] CR2: 0000000000942d58 CR3: 0000000005c26000 CR4: 00000000000006e0 [ 293.110057] DR0: 000000000089e000 DR1: 00007f938d51a000 DR2: 0000000000000000 [ 293.110057] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 293.110057] Stack: [ 293.110057] ffff8805b91d0be0 00000000c40bb9d8 ffff8805b82c1cf8 ffff8805b91d0be0 [ 293.110057] ffff8800c40bba28 ffffffff812037d0 ffff8800c40bb978 ffff8805b91d0c30 [ 293.110057] ffff8805b82c1de8 ffffffff81202f56 0000000000000001 ffffffff81202e44 [ 293.110057] Call Trace: [ 293.110057] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 293.110057] [<ffffffff81202f56>] cpu_stopper_thread+0x96/0x190 [ 293.110057] [<ffffffff81202e44>] ? cpu_stop_should_run+0x44/0x60 [ 293.110057] [<ffffffff811a3d1a>] ? __lock_release+0x1da/0x1f0 [ 293.110057] [<ffffffff84384ec5>] ? _raw_spin_unlock_irqrestore+0x65/0xb0 [ 293.110057] [<ffffffff811a1f0d>] ? trace_hardirqs_on+0xd/0x10 [ 293.110057] [<ffffffff81170826>] smpboot_thread_fn+0x2b6/0x2c0 [ 293.110057] [<ffffffff81170570>] ? smpboot_register_percpu_thread+0x100/0x100 [ 293.110057] [<ffffffff81167755>] kthread+0x105/0x110 [ 293.110057] [<ffffffff8437fca2>] ? wait_for_completion+0xc2/0x110 [ 293.110057] [<ffffffff81167650>] ? set_kthreadd_affinity+0x30/0x30 [ 293.110057] [<ffffffff8438e28c>] ret_from_fork+0x7c/0xb0 [ 293.110057] [<ffffffff81167650>] ? set_kthreadd_affinity+0x30/0x30 [ 293.110057] Code: b5 05 01 66 0f 1f 44 00 00 45 89 ed 4d 0f a3 2e 45 19 ed 45 85 ed 41 0f 95 c7 4c 8d 73 24 31 c0 c7 45 cc 00 00 00 00 66 90 f3 90 <44> 8b 6b 20 41 39 c5 74 66 41 83 fd 02 74 0f 41 83 fd 03 75 41
This is followed by a dump of the rest of the CPU states, and I see the following:
[ 268.450444] NMI backtrace for cpu 34 [ 268.450444] CPU: 34 PID: 9859 Comm: trinity-c129 Tainted: G W 3.14.0-rc3-next-20140218-sasha-00008-g16140ff-dirty #109 [ 268.450444] task: ffff880154aa3000 ti: ffff8800c40ba000 task.ti: ffff8800c40ba000 [ 268.450444] RIP: 0010:[<ffffffff810b801a>] [<ffffffff810b801a>] pvclock_clocksource_read+0x1a/0xc0 [ 268.450444] RSP: 0018:ffff8800c40bb538 EFLAGS: 00000082 [ 268.450444] RAX: 00000000905f62ec RBX: ffff8805b93d8500 RCX: 000000000000000a [ 268.450444] RDX: 00000000000000a1 RSI: 0000000000000040 RDI: ffff881027f12880 [ 268.450444] RBP: ffff8800c40bb548 R08: 0000000000000000 R09: 0000000000000001 [ 268.450444] R10: 0000000000000002 R11: 0000000000000001 R12: ffff8805b93d8510 [ 268.450444] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000001 [ 268.450444] FS: 00007f938e16a700(0000) GS:ffff8805b9200000(0000) knlGS:0000000000000000 [ 268.450444] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 268.450444] CR2: 00007f938a658a38 CR3: 00000000c40b2000 CR4: 00000000000006e0 [ 268.450444] DR0: 000000000089e000 DR1: 00007f938d51a000 DR2: 0000000000000000 [ 268.450444] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000f7060a [ 268.450444] Stack: [ 268.450444] ffff8805b93d8500 ffff8805b93d8510 ffff8800c40bb558 ffffffff810b6f04 [ 268.450444] ffff8800c40bb568 ffffffff81076e1d ffff8800c40bb598 ffffffff81180635 [ 268.450444] ffffffff875f7be0 0000000000000022 00000000001d8500 ffff8805b93d8500 [ 268.450444] Call Trace: [ 268.450444] [<ffffffff810b6f04>] kvm_clock_read+0x24/0x50 [ 268.450444] [<ffffffff81076e1d>] sched_clock+0x1d/0x30 [ 268.450444] [<ffffffff81180635>] sched_clock_local+0x25/0x90 [ 268.450444] [<ffffffff811808a8>] sched_clock_cpu+0xb8/0x110 [ 268.450444] [<ffffffff8118094d>] local_clock+0x2d/0x40 [ 268.450444] [<ffffffff811a3358>] __lock_acquire+0x2a8/0x580 [ 268.450444] [<ffffffff8118c84d>] ? update_blocked_averages+0x61d/0x670 [ 268.450444] [<ffffffff811a37b2>] lock_acquire+0x182/0x1d0 [ 268.450444] [<ffffffff8118ce2c>] ? idle_balance+0x3c/0x2c0 [ 268.450444] [<ffffffff8118ce61>] idle_balance+0x71/0x2c0 [ 268.450444] [<ffffffff8118ce2c>] ? idle_balance+0x3c/0x2c0 [ 268.450444] [<ffffffff8118d2fe>] pick_next_task_fair+0x24e/0x290 [ 268.450444] [<ffffffff81173a2e>] ? dequeue_task+0x6e/0x80 [ 268.450444] [<ffffffff8437e3b5>] __schedule+0x2a5/0x840 [ 268.450444] [<ffffffff8437ec15>] schedule+0x65/0x70 [ 268.450444] [<ffffffff8437d8d8>] schedule_timeout+0x38/0x2a0 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff811a196c>] ? mark_held_locks+0x6c/0x90 [ 268.450444] [<ffffffff84384e0b>] ? _raw_spin_unlock_irq+0x2b/0x80 [ 268.450444] [<ffffffff8437fc97>] wait_for_completion+0xb7/0x110 [ 268.450444] [<ffffffff8117e6b0>] ? try_to_wake_up+0x2a0/0x2a0 [ 268.450444] [<ffffffff812034a1>] stop_two_cpus+0x261/0x2b0 [ 268.450444] [<ffffffff81176c00>] ? __migrate_swap_task+0xa0/0xa0 [ 268.450444] [<ffffffff810b6f04>] ? kvm_clock_read+0x24/0x50 [ 268.450444] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 268.450444] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff8437fc1f>] ? wait_for_completion+0x3f/0x110 [ 268.450444] [<ffffffff811740d5>] migrate_swap+0x175/0x1a0 [ 268.450444] [<ffffffff8118b1e0>] task_numa_migrate+0x510/0x600 [ 268.450444] [<ffffffff8118acd0>] ? select_task_rq_fair+0x440/0x440 [ 268.450444] [<ffffffff8118b31d>] numa_migrate_preferred+0x4d/0x60 [ 268.450444] [<ffffffff8118b900>] task_numa_fault+0x190/0x210 [ 268.450444] [<ffffffff812c4f62>] do_huge_pmd_numa_page+0x432/0x450 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff81abccda>] ? delay_tsc+0xea/0x110 [ 268.450444] [<ffffffff81288724>] __handle_mm_fault+0x244/0x3a0 [ 268.450444] [<ffffffff812c776d>] ? rcu_read_unlock+0x5d/0x60 [ 268.450444] [<ffffffff8128898b>] handle_mm_fault+0x10b/0x1b0 [ 268.450444] [<ffffffff84389252>] ? __do_page_fault+0x2e2/0x590 [ 268.450444] [<ffffffff843894c1>] __do_page_fault+0x551/0x590 [ 268.450444] [<ffffffff81181561>] ? vtime_account_user+0x91/0xa0 [ 268.450444] [<ffffffff8124f0c8>] ? context_tracking_user_exit+0xa8/0x1c0 [ 268.450444] [<ffffffff84384f40>] ? _raw_spin_unlock+0x30/0x50 [ 268.450444] [<ffffffff81181561>] ? vtime_account_user+0x91/0xa0 [ 268.450444] [<ffffffff8124f0c8>] ? context_tracking_user_exit+0xa8/0x1c0 [ 268.450444] [<ffffffff843895bd>] do_page_fault+0x3d/0x70 [ 268.450444] [<ffffffff843888b5>] do_async_page_fault+0x35/0x100 [ 268.450444] [<ffffffff84385798>] async_page_fault+0x28/0x30 [ 268.450444] Code: 6d e1 10 00 e8 98 8a 15 00 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 48 83 ec 10 8b 37 eb 06 0f 1f 40 00 89 c6 0f 1f 00 0f ae e8 0f 31 <48> c1 e2 20 89 c0 0f be 4f 1c 48 09 c2 44 8b 47 18 48 2b 57 08
The stack trace is always pointing to migrate_swap() calling stop_two_cpus(). Note that I don't see another CPU in a stopped state when it occurs.
Thanks,
Sasha
Quoted 4 lines · Sasha Levin
Hi all,
While fuzzing with trinity inside a KVM tools guest, running latest -next kernel, I see to hit the following hang quite often.
Fix for the stuck issue around idle_balance() is now in progress, this may caused be the same problem, I suggest we do some retest after these patch got merged.
Regards,
Michael Wang
Quoted 173 lines
The initial spew is:
[ 293.110057] BUG: soft lockup - CPU#8 stuck for 22s! [migration/8:258] [ 293.110057] Modules linked in: [ 293.110057] irq event stamp: 20828 [ 293.110057] hardirqs last enabled at (20827): [<ffffffff84385437>] restore_args+0x0/0x30 [ 293.110057] hardirqs last disabled at (20828): [<ffffffff8438f02d>] apic_timer_interrupt+0x6d/0x80 [ 293.110057] softirqs last enabled at (20826): [<ffffffff811418e7>] __do_softirq+0x447/0x4f0 [ 293.110057] softirqs last disabled at (20821): [<ffffffff81141fa3>] irq_exit+0x83/0x160 [ 293.110057] CPU: 8 PID: 258 Comm: migration/8 Tainted: G W 3.14.0-rc3-next-20140218-sasha-00008-g16140ff-dirty #109 [ 293.110057] task: ffff8805b8283000 ti: ffff8805b82c0000 task.ti: ffff8805b82c0000 [ 293.110057] RIP: 0010:[<ffffffff81203892>] [<ffffffff81203892>] multi_cpu_stop+0xc2/0x1c0 [ 293.110057] RSP: 0000:ffff8805b82c1cd8 EFLAGS: 00000293 [ 293.110057] RAX: 0000000000000001 RBX: ffffffff84385437 RCX: 0000000000000000 [ 293.110057] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8800c40bb978 [ 293.110057] RBP: ffff8805b82c1d18 R08: 0000000000000000 R09: 0000000000000000 [ 293.110057] R10: 0000000000000001 R11: 0000000000000001 R12: ffff8805b82c1c48 [ 293.110057] R13: ffff8805b8283000 R14: ffff8805b82c0000 R15: ffff8805b8283000 [ 293.110057] FS: 0000000000000000(0000) GS:ffff8805b9000000(0000) knlGS:0000000000000000 [ 293.110057] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 293.110057] CR2: 0000000000942d58 CR3: 0000000005c26000 CR4: 00000000000006e0 [ 293.110057] DR0: 000000000089e000 DR1: 00007f938d51a000 DR2: 0000000000000000 [ 293.110057] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 293.110057] Stack: [ 293.110057] ffff8805b91d0be0 00000000c40bb9d8 ffff8805b82c1cf8 ffff8805b91d0be0 [ 293.110057] ffff8800c40bba28 ffffffff812037d0 ffff8800c40bb978 ffff8805b91d0c30 [ 293.110057] ffff8805b82c1de8 ffffffff81202f56 0000000000000001 ffffffff81202e44 [ 293.110057] Call Trace: [ 293.110057] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 293.110057] [<ffffffff81202f56>] cpu_stopper_thread+0x96/0x190 [ 293.110057] [<ffffffff81202e44>] ? cpu_stop_should_run+0x44/0x60 [ 293.110057] [<ffffffff811a3d1a>] ? __lock_release+0x1da/0x1f0 [ 293.110057] [<ffffffff84384ec5>] ? _raw_spin_unlock_irqrestore+0x65/0xb0 [ 293.110057] [<ffffffff811a1f0d>] ? trace_hardirqs_on+0xd/0x10 [ 293.110057] [<ffffffff81170826>] smpboot_thread_fn+0x2b6/0x2c0 [ 293.110057] [<ffffffff81170570>] ? smpboot_register_percpu_thread+0x100/0x100 [ 293.110057] [<ffffffff81167755>] kthread+0x105/0x110 [ 293.110057] [<ffffffff8437fca2>] ? wait_for_completion+0xc2/0x110 [ 293.110057] [<ffffffff81167650>] ? set_kthreadd_affinity+0x30/0x30 [ 293.110057] [<ffffffff8438e28c>] ret_from_fork+0x7c/0xb0 [ 293.110057] [<ffffffff81167650>] ? set_kthreadd_affinity+0x30/0x30 [ 293.110057] Code: b5 05 01 66 0f 1f 44 00 00 45 89 ed 4d 0f a3 2e 4519 ed 45 85 ed 41 0f 95 c7 4c 8d 73 24 31 c0 c7 45 cc 00 00 00 00 66 90 f3 90 <44> 8b 6b 20 41 39 c5 74 66 41 83 fd 02 74 0f 41 83 fd 03 75 41
This is followed by a dump of the rest of the CPU states, and I see the following:
[ 268.450444] NMI backtrace for cpu 34 [ 268.450444] CPU: 34 PID: 9859 Comm: trinity-c129 Tainted: G W 3.14.0-rc3-next-20140218-sasha-00008-g16140ff-dirty #109 [ 268.450444] task: ffff880154aa3000 ti: ffff8800c40ba000 task.ti: ffff8800c40ba000 [ 268.450444] RIP: 0010:[<ffffffff810b801a>] [<ffffffff810b801a>] pvclock_clocksource_read+0x1a/0xc0 [ 268.450444] RSP: 0018:ffff8800c40bb538 EFLAGS: 00000082 [ 268.450444] RAX: 00000000905f62ec RBX: ffff8805b93d8500 RCX: 000000000000000a [ 268.450444] RDX: 00000000000000a1 RSI: 0000000000000040 RDI: ffff881027f12880 [ 268.450444] RBP: ffff8800c40bb548 R08: 0000000000000000 R09: 0000000000000001 [ 268.450444] R10: 0000000000000002 R11: 0000000000000001 R12: ffff8805b93d8510 [ 268.450444] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000001 [ 268.450444] FS: 00007f938e16a700(0000) GS:ffff8805b9200000(0000) knlGS:0000000000000000 [ 268.450444] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 268.450444] CR2: 00007f938a658a38 CR3: 00000000c40b2000 CR4: 00000000000006e0 [ 268.450444] DR0: 000000000089e000 DR1: 00007f938d51a000 DR2: 0000000000000000 [ 268.450444] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000f7060a [ 268.450444] Stack: [ 268.450444] ffff8805b93d8500 ffff8805b93d8510 ffff8800c40bb558 ffffffff810b6f04 [ 268.450444] ffff8800c40bb568 ffffffff81076e1d ffff8800c40bb598 ffffffff81180635 [ 268.450444] ffffffff875f7be0 0000000000000022 00000000001d8500 ffff8805b93d8500 [ 268.450444] Call Trace: [ 268.450444] [<ffffffff810b6f04>] kvm_clock_read+0x24/0x50 [ 268.450444] [<ffffffff81076e1d>] sched_clock+0x1d/0x30 [ 268.450444] [<ffffffff81180635>] sched_clock_local+0x25/0x90 [ 268.450444] [<ffffffff811808a8>] sched_clock_cpu+0xb8/0x110 [ 268.450444] [<ffffffff8118094d>] local_clock+0x2d/0x40 [ 268.450444] [<ffffffff811a3358>] __lock_acquire+0x2a8/0x580 [ 268.450444] [<ffffffff8118c84d>] ? update_blocked_averages+0x61d/0x670 [ 268.450444] [<ffffffff811a37b2>] lock_acquire+0x182/0x1d0 [ 268.450444] [<ffffffff8118ce2c>] ? idle_balance+0x3c/0x2c0 [ 268.450444] [<ffffffff8118ce61>] idle_balance+0x71/0x2c0 [ 268.450444] [<ffffffff8118ce2c>] ? idle_balance+0x3c/0x2c0 [ 268.450444] [<ffffffff8118d2fe>] pick_next_task_fair+0x24e/0x290 [ 268.450444] [<ffffffff81173a2e>] ? dequeue_task+0x6e/0x80 [ 268.450444] [<ffffffff8437e3b5>] __schedule+0x2a5/0x840 [ 268.450444] [<ffffffff8437ec15>] schedule+0x65/0x70 [ 268.450444] [<ffffffff8437d8d8>] schedule_timeout+0x38/0x2a0 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff811a196c>] ? mark_held_locks+0x6c/0x90 [ 268.450444] [<ffffffff84384e0b>] ? _raw_spin_unlock_irq+0x2b/0x80 [ 268.450444] [<ffffffff8437fc97>] wait_for_completion+0xb7/0x110 [ 268.450444] [<ffffffff8117e6b0>] ? try_to_wake_up+0x2a0/0x2a0 [ 268.450444] [<ffffffff812034a1>] stop_two_cpus+0x261/0x2b0 [ 268.450444] [<ffffffff81176c00>] ? __migrate_swap_task+0xa0/0xa0 [ 268.450444] [<ffffffff810b6f04>] ? kvm_clock_read+0x24/0x50 [ 268.450444] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 268.450444] [<ffffffff812037d0>] ? stop_cpus+0x60/0x60 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff8437fc1f>] ? wait_for_completion+0x3f/0x110 [ 268.450444] [<ffffffff811740d5>] migrate_swap+0x175/0x1a0 [ 268.450444] [<ffffffff8118b1e0>] task_numa_migrate+0x510/0x600 [ 268.450444] [<ffffffff8118acd0>] ? select_task_rq_fair+0x440/0x440 [ 268.450444] [<ffffffff8118b31d>] numa_migrate_preferred+0x4d/0x60 [ 268.450444] [<ffffffff8118b900>] task_numa_fault+0x190/0x210 [ 268.450444] [<ffffffff812c4f62>] do_huge_pmd_numa_page+0x432/0x450 [ 268.450444] [<ffffffff8119c8fe>] ? put_lock_stats+0xe/0x30 [ 268.450444] [<ffffffff81abccda>] ? delay_tsc+0xea/0x110 [ 268.450444] [<ffffffff81288724>] __handle_mm_fault+0x244/0x3a0 [ 268.450444] [<ffffffff812c776d>] ? rcu_read_unlock+0x5d/0x60 [ 268.450444] [<ffffffff8128898b>] handle_mm_fault+0x10b/0x1b0 [ 268.450444] [<ffffffff84389252>] ? __do_page_fault+0x2e2/0x590 [ 268.450444] [<ffffffff843894c1>] __do_page_fault+0x551/0x590 [ 268.450444] [<ffffffff81181561>] ? vtime_account_user+0x91/0xa0 [ 268.450444] [<ffffffff8124f0c8>] ? context_tracking_user_exit+0xa8/0x1c0 [ 268.450444] [<ffffffff84384f40>] ? _raw_spin_unlock+0x30/0x50 [ 268.450444] [<ffffffff81181561>] ? vtime_account_user+0x91/0xa0 [ 268.450444] [<ffffffff8124f0c8>] ? context_tracking_user_exit+0xa8/0x1c0 [ 268.450444] [<ffffffff843895bd>] do_page_fault+0x3d/0x70 [ 268.450444] [<ffffffff843888b5>] do_async_page_fault+0x35/0x100 [ 268.450444] [<ffffffff84385798>] async_page_fault+0x28/0x30 [ 268.450444] Code: 6d e1 10 00 e8 98 8a 15 00 c9 c3 66 0f 1f 44 00 0055 48 89 e5 48 83 ec 10 8b 37 eb 06 0f 1f 40 00 89 c6 0f 1f 00 0f ae e8 0f 31 <48> c1 e2 20 89 c0 0f be 4f 1c 48 09 c2 44 8b 47 18 48 2b 57 08
The stack trace is always pointing to
migrate_swap()callingstop_two_cpus(). Note that I don't
see another CPU in a stopped state when it occurs.Thanks,
SashaSignature · 4 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Quoted 8 lines · Michael wang
Quoted 4 lines · Sasha Levin
Quoted 4 lines
Hi all,
While fuzzing with trinity inside a KVM tools guest, running latest -next kernel, I see to hit the following hang quite often.
Fix for the stuck issue around
idle_balance()is now in progress, this may caused be the same problem, I suggest we do some retest after these patch got merged.
Could I get a link to the patch please? It's a pain testing other things with this issue reproducing every time.
Thanks,
Sasha
Quoted 4 lines · Sasha Levin
Quoted 3 lines · Michael wang
Quoted 2 lines · Sasha Levin
Quoted 2 lines
Hi all,
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
While fuzzing with trinity inside a KVM tools guest, running latest -next kernel, I see to hit the following hang quite often.
Fix for the stuck issue around
idle_balance()is now in progress, this may caused be the same problem, I suggest we do some retest after these patch got merged.Could I get a link to the patch please? It's a pain testing other things with this issue reproducing every time.
Please take a try on the latest tip tree, patches has been merged as I saw :)
Regards,
Michael Wang
Quoted 10 lines
Thanks,
SashaSignature · 4 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Quoted 11 lines · Michael wang
Quoted 10 lines · Sasha Levin
Quoted 8 lines · Michael wang
Quoted 4 lines · Sasha Levin
Quoted 4 lines
Hi all,
While fuzzing with trinity inside a KVM tools guest, running latest -next kernel, I see to hit the following hang quite often.
Fix for the stuck issue around
idle_balance()is now in progress, this may caused be the same problem, I suggest we do some retest after these patch got merged.
Quoted 5 lines
Quoted 2 lines
Could I get a link to the patch please? It's a pain testing other things with this issue reproducing every time.
Please take a try on the latest tip tree, patches has been merged as I saw :)
Nope, still see it with latest -tip.
I ran tip's master branch, should I have tried a different one?
Thanks,
sasha
On 02/24/2014 11:23 AM, Sasha Levin wrote:
[snip]
Quoted 10 lines
Quoted 6 lines
Quoted 3 lines
Could I get a link to the patch please? It's a pain testing other things with this issue reproducing every time.
Please take a try on the latest tip tree, patches has been merged as I saw :)
Nope, still see it with latest -tip.
I ran tip's master branch, should I have tried a different one?
Hmm... I don't see the changes we expected on master either...
Peter, do we accidentally missed this commit?
http://git.kernel.org/tip/477af336ba06ef4c32e97892bb0d2027ce30f466
Regards,
Michael Wang
Quoted 11 lines
Thanks,
sashaSignature · 4 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
Quoted 9 lines · Michael wang
On 02/24/2014 11:23 AM, Sasha Levin wrote:
[snip]Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
Quoted 3 lines
Could I get a link to the patch please? It's a pain testing other things with this issue reproducing every time.
Please take a try on the latest tip tree, patches has been merged as I saw:)
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Nope, still see it with latest -tip.
I ran tip's master branch, should I have tried a different one?
Hmm... I don't see the changes we expected on master either...
Peter, do we accidentally missed this commit?
http://git.kernel.org/tip/477af336ba06ef4c32e97892bb0d2027ce30f466
I've tried grabbing this patch and it does pretty bad things on my testing VM, it just gets stuck early in the boot process and doesn't show any progress. The VM never even gets close to starting up all the cpus (usually dies around the lockdep self-testing).
Are there any other dependencies?
Thanks,
Sasha
Peter Zijlstra
maintainerQuoted 3 lines · Michael wang
Peter, do we accidentally missed this commit?
http://git.kernel.org/tip/477af336ba06ef4c32e97892bb0d2027ce30f466
Ingo dropped it on Saturday because it makes locking_selftest() unhappy.
That is because we call locking_selftest() way before we're ready to call schedule() and guess what it does :-/
I'm not entirely sure what to do.. ideally I'd shoot locking_selftest in the head, but clearly that's not entirely desired either.
Quoted 5 lines · Peter Zijlstra
Quoted 3 lines · Michael wang
Peter, do we accidentally missed this commit?
http://git.kernel.org/tip/477af336ba06ef4c32e97892bb0d2027ce30f466
Quoted 7 lines
Ingo dropped it on Saturday because it makes
locking_selftest()unhappy.That is because we call
locking_selftest()way before we're ready to callschedule()and guess what it does :-/I'm not entirely sure what to do.. ideally I'd shoot
locking_selftestin the head, but clearly that's not entirely desired either.
...what about move idle_balance() back to it's old position?
pull_rt_task() logical could be after idle_balance() if still no FAIR and DL, then go into the pick loop, that may could make things more clean & clear, should we have a try?
Regards,
Michael Wang
Quoted 6 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Peter Zijlstra
maintainerQuoted 8 lines · Michael wang
Quoted 7 lines · Peter Zijlstra
Quoted 3 lines · Michael wang
Peter, do we accidentally missed this commit?
http://git.kernel.org/tip/477af336ba06ef4c32e97892bb0d2027ce30f466
Ingo dropped it on Saturday because it makes
locking_selftest()unhappy.
Quoted 7 lines
Quoted 5 lines
That is because we call
locking_selftest()way before we're ready to callschedule()and guess what it does :-/I'm not entirely sure what to do.. ideally I'd shoot
locking_selftestin the head, but clearly that's not entirely desired either....what about move
idle_balance()back to it's old position?
I've always hated that, idle_balance() is very much a fair policy thing and shouldn't live in the core code.
Quoted 3 lines
pull_rt_task()logical could be afteridle_balance()if still no FAIR and DL, then go into the pick loop, that may could make things more clean & clear, should we have a try?
So the reason pull_{rt,dl}_task() is before idle_balance() is that we don't want to add the execution latency of idle_balance() to the rt/dl task pulling.
Anyway, the below seems to work; it avoids playing tricks with the idle thread and instead uses a magic constant.
The comparison should be faster too; seeing how we avoid dereferencing p->sched_class.
---
Subject: sched: Guarantee task priority in pick_next_task()
From: Peter Zijlstra <[email protected]>
Date: Fri Feb 14 12:25:08 CET 2014
Michael spotted that the idle_balance() push down created a task priority problem.
Previously, when we called idle_balance() before pick_next_task() it wasn't a problem when -- because of the rq->lock droppage -- an rt/dl task slipped in.
Similarly for pre_schedule(), rt pre-schedule could have a dl task slip in.
But by pulling it into the pick_next_task() loop, we'll not try a higher task priority again.
Cure this by creating a re-start condition in pick_next_task(); and triggering this from pick_next_task_{rt,fair}().
--- kernel/sched/core.c | 12 ++++++++---- kernel/sched/fair.c | 13 ++++++++++++- kernel/sched/rt.c | 10 +++++++++- kernel/sched/sched.h | 5 +++++ 4 files changed, 34 insertions(+), 6 deletions(-)
kernel/sched/core.c +8 -4
@@ -2586,24 +2586,28 @@ static inline void schedule_debug(structstatic inline struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev) { -const struct sched_class *class; +const struct sched_class *class = &fair_sched_class;struct task_struct *p;/* * Optimization: we know that if all tasks are in * the fair class we can call that function directly: */ -if (likely (prev->sched_class == &fair_sched_class && +if (likely (prev->sched_class == class && rq->nr_running == rq->cfs.h_nr_running)) { p = fair_sched_class.pick_next_task (rq, prev); -if (likely (p)) +if (likely (p && p !=RETRY_TASK ))return p; } +again:for_each_class (class) { p = class->pick_next_task (rq, prev); -if (p) +if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;return p; + } }BUG ();/* the idle class will always have a runnable task */
kernel/sched/fair.c +12 -1
@@ -4687,6 +4687,7 @@ pick_next_task_fair(struct rq *rq, strucstruct cfs_rq *cfs_rq = &rq->cfs;struct sched_entity *se;struct task_struct *p; +int new_tasks; again: #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -4785,7 +4786,17 @@ pick_next_task_fair(struct rq *rq, strucreturn p; idle: -if (idle_balance (rq))/* drops rq->lock */ +/* +* Because idle_balance() releases (and re-acquires) rq->lock, it is +* possible for any higher priority task to appear. In that case we +* must re-start the pick_next_entity() loop. +*/ + new_tasks =idle_balance (rq); + +if (rq->nr_running != rq->cfs.h_nr_running) +return RETRY_TASK ; + +if (new_tasks) goto again;return NULL ;
kernel/sched/rt.c +9 -1
@@ -1360,8 +1360,16 @@ pick_next_task_rt(struct rq *rq, structstruct task_struct *p;struct rt_rq *rt_rq = &rq->rt; -if (need_pull_rt_task (rq, prev)) +if (need_pull_rt_task (rq, prev)) {pull_rt_task (rq); +/* +* pull_rt_task() can drop (and re-acquire) rq->lock; this +* means a dl task can slip in, in which case we need to +* re-start task selection. +*/ +if (unlikely (rq->dl.dl_nr_running)) +return RETRY_TASK ; + }if (!rt_rq->rt_nr_running)return NULL ;
kernel/sched/sched.h +5 -0
@@ -1090,6 +1090,8 @@ static const u32 prio_to_wmult[40] = {#define DEQUEUE_SLEEP 1 +#define RETRY_TASK ((void *)-1UL) +struct sched_class {const struct sched_class *next;
@@ -1104,6 +1106,9 @@ struct sched_class {* It is the responsibility of the pick_next_task() method that will * return the next task to call put_prev_task() on the @prev task or * something equivalent. +* +* May return RETRY_TASK when it finds a higher prio class has runnable +* tasks. */ struct task_struct * (*pick_next_task) (struct rq *rq,struct task_struct *prev);
Peter Zijlstra
maintainerkernel/sched/core.c static inline void schedule_debug(struct · line 2608
Quoted 3 lines · Peter Zijlstra
kernel/sched/core.c +3
-0+if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;
We could even make that: unlikely(p & 1), I think most CPUs can encode that far better than the full pointer immediate.
Quoted 27 lines · Peter Zijlstra
Anyway, the below seems to work; it avoids playing tricks with the idle thread and instead uses a magic constant.
The comparison should be faster too; seeing how we avoid dereferencing
p->sched_class.---
Subject: sched: Guarantee task priority inpick_next_task()
From: Peter Zijlstra<[email protected]>
Date: Fri Feb 14 12:25:08 CET 2014Michael spotted that the
idle_balance()push down created a task priority problem.Previously, when we called
idle_balance()beforepick_next_task()it wasn't a problem when -- because of therq->lockdroppage -- an rt/dl task slipped in.Similarly for
pre_schedule(), rt pre-schedule could have a dl task slip in.But by pulling it into the
pick_next_task()loop, we'll not try a higher task priority again.Cure this by creating a re-start condition in
pick_next_task(); and triggering this from pick_next_task_{rt,fair}().
Quoted 6 lines
Fixes 38033c37faab ("sched: Push down pre_schedule() and idle_balance()")Cc Juri Lelli<[email protected]>Cc Ingo Molnar<[email protected]>Cc Steven Rostedt<[email protected]>Reported-by Michael Wang<[email protected]>Signed-off-by Peter Zijlstra<[email protected]>
Sign me up to the fan club of this patch, I love it.
After running it for a bit I see that all the hangs I've seen before are gone, even those that I didn't assume were sched related.
Thanks,
Sasha
On 02/25/2014 02:21 AM, Sasha Levin wrote:
[snip]
Quoted 9 lines
Quoted 8 lines
Fixes 38033c37faab ("sched: Push down pre_schedule() and
idle_balance()")Cc Juri Lelli<[email protected]>Cc Ingo Molnar<[email protected]>Cc Steven Rostedt<[email protected]>Reported-by Michael Wang<[email protected]>Signed-off-by Peter Zijlstra<[email protected]>
Quoted 4 lines
Sign me up to the fan club of this patch, I love it.
After running it for a bit I see that all the hangs I've seen before are gone, even those that I didn't assume were sched related.
Good to know the problem got solved :)
Regards,
Michael Wang
Quoted 4 lines
Thanks,
Sasha
On 02/24/2014 08:12 PM, Peter Zijlstra wrote:
[snip]
Quoted 6 lines
Quoted 2 lines
...what about move
idle_balance()back to it's old position?I've always hated that,
idle_balance()is very much a fair policy thing and shouldn't live in the core code.
Quoted 7 lines
Quoted 3 lines
pull_rt_task()logical could be afteridle_balance()if still no FAIR and DL, then go into the pick loop, that may could make things more clean & clear, should we have a try?So the reason pull_{rt,dl}_task() is before
idle_balance()is that we don't want to add the execution latency ofidle_balance()to the rt/dl task pulling.
Yeah, that make sense, just wondering... since RT also has balance stuff, may be we can use a new call back for each class in the old position?
The new idle_balance could like:
void idle_balance() {
for_each_class(class)
if class->idle_balance()
break
}Quoted 6 lines
Anyway, the below seems to work; it avoids playing tricks with the idle thread and instead uses a magic constant.
The comparison should be faster too; seeing how we avoid dereferencing
p->sched_class.
Great, it once appeared in my mind but you achieved this without new parameter, now let's ignore my wondering above :)
Regards,
Michael Wang
Quoted 146 lines
---
Subject: sched: Guarantee task priority inpick_next_task()
From: Peter Zijlstra <[email protected]>
Date: Fri Feb 14 12:25:08 CET 2014Michael spotted that the
idle_balance()push down created a task priority problem.Previously, when we called
idle_balance()beforepick_next_task()it wasn't a problem when -- because of therq->lockdroppage -- an rt/dl task slipped in.Similarly for
pre_schedule(), rt pre-schedule could have a dl task slip in.But by pulling it into the
pick_next_task()loop, we'll not try a higher task priority again.Cure this by creating a re-start condition in
pick_next_task(); and triggering this from pick_next_task_{rt,fair}().Fixes 38033c37faab ("sched: Push down pre_schedule() and idle_balance()")Cc Juri Lelli <[email protected]>Cc Ingo Molnar <[email protected]>Cc Steven Rostedt <[email protected]>Reported-by Michael Wang <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>--- kernel/sched/core.c | 12 ++++++++---- kernel/sched/fair.c | 13 ++++++++++++- kernel/sched/rt.c | 10 +++++++++- kernel/sched/sched.h | 5 +++++ 4 files changed, 34 insertions(+), 6 deletions(-)kernel/sched/core.c +8
-4@@ -2586,24 +2586,28 @@ static inline void schedule_debug(structstatic inline struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev) { -const struct sched_class *class; +const struct sched_class *class = &fair_sched_class;struct task_struct *p;/* * Optimization: we know that if all tasks are in * the fair class we can call that function directly: */ -if (likely (prev->sched_class == &fair_sched_class && +if (likely (prev->sched_class == class && rq->nr_running == rq->cfs.h_nr_running)) { p = fair_sched_class.pick_next_task (rq, prev); -if (likely (p)) +if (likely (p && p !=RETRY_TASK ))return p; } +again:for_each_class (class) { p = class->pick_next_task (rq, prev); -if (p) +if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;return p; + } }BUG ();/* the idle class will always have a runnable task */ kernel/sched/fair.c +12
-1@@ -4687,6 +4687,7 @@ pick_next_task_fair(struct rq *rq, strucstruct cfs_rq *cfs_rq = &rq->cfs;struct sched_entity *se;struct task_struct *p; +int new_tasks; again: #ifdef CONFIG_FAIR_GROUP_SCHED@@ -4785,7 +4786,17 @@ pick_next_task_fair(struct rq *rq, strucreturn p; idle: -if (idle_balance (rq))/* drops rq->lock */ +/* +* Because idle_balance() releases (and re-acquires) rq->lock, it is +* possible for any higher priority task to appear. In that case we +* must re-start the pick_next_entity() loop. +*/ + new_tasks =idle_balance (rq); + +if (rq->nr_running != rq->cfs.h_nr_running) +return RETRY_TASK ; + +if (new_tasks) goto again;return NULL ;kernel/sched/rt.c +9
-1@@ -1360,8 +1360,16 @@ pick_next_task_rt(struct rq *rq, structstruct task_struct *p;struct rt_rq *rt_rq = &rq->rt; -if (need_pull_rt_task (rq, prev)) +if (need_pull_rt_task (rq, prev)) {pull_rt_task (rq); +/* +* pull_rt_task() can drop (and re-acquire) rq->lock; this +* means a dl task can slip in, in which case we need to +* re-start task selection. +*/ +if (unlikely (rq->dl.dl_nr_running)) +return RETRY_TASK ; + }if (!rt_rq->rt_nr_running)return NULL ;kernel/sched/sched.h +5
-1@@ -1090,6 +1090,8 @@ static const u32 prio_to_wmult[40] = {#define DEQUEUE_SLEEP 1 +#define RETRY_TASK ((void *)-1UL) +struct sched_class {const struct sched_class *next;@@ -1104,6 +1106,9 @@ struct sched_class {* It is the responsibility of the pick_next_task() method that will * return the next task to call put_prev_task() on the @prev task or * something equivalent. +* +* May return RETRY_TASK when it finds a higher prio class has runnable +* tasks. */ struct task_struct * (*pick_next_task) (struct rq *rq,struct task_struct *prev); --To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Quoted 7 lines · Peter Zijlstra
kernel/sched/core.c static inline void schedule_debug(struct · line 2608
Quoted 3 lines · Peter Zijlstra
kernel/sched/core.c +3
-0+if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;We could even make that:
unlikely(p & 1), I think most CPUs can encode that far better than the full pointer immediate.
Agree, unless odd-align stuff appeared...
Regards,
Michael Wang
Quoted 6 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Peter Zijlstra
maintainerQuoted 10 lines · Michael wang
Quoted 7 lines · Peter Zijlstra
kernel/sched/core.c static inline void schedule_debug(struct · line 2608
Quoted 3 lines · Peter Zijlstra
kernel/sched/core.c +3
-0+if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;We could even make that:
unlikely(p & 1), I think most CPUs can encode that far better than the full pointer immediate.Agree, unless odd-align stuff appeared...
That shouldn't happen; we rely on u32 alignment all over the place. So for general objects we should have at least the lower two bits free to play with.
Peter Zijlstra
maintainerQuoted 1 line · Sasha Levin
Sign me up to the fan club of this patch, I love it.
I've converted that in a Tested-by: tag :-)
Quoted 10 lines · Peter Zijlstra
Quoted 9 lines · Michael wang
Quoted 7 lines · Peter Zijlstra
kernel/sched/core.c static inline void schedule_debug(struct · line 2608
Quoted 3 lines · Peter Zijlstra
kernel/sched/core.c +3
-0+if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;We could even make that:
unlikely(p & 1), I think most CPUs can encode that far better than the full pointer immediate.
Quoted 5 lines
Quoted 1 line
Agree, unless odd-align stuff appeared...
That shouldn't happen; we rely on u32 alignment all over the place. So for general objects we should have at least the lower two bits free to play with.
Make sense :)
Regards,
Michael Wang
Quoted 6 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
tip-bot for Peter Zijlstra
merge bot[tip:sched/core] sched: Guarantee task priority in pick_next_task ()
In reply to Peter Zijlstrasched: Guarantee task priority in pick_next_task()
Commit-ID: 37e117c07b89194aae7062bc63bde1104c03db02 Gitweb: http://git.kernel.org/tip/37e117c07b89194aae7062bc63bde1104c03db02 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 14 Feb 2014 12:25:08 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 27 Feb 2014 12:41:02 +0100
sched: Guarantee task priority in pick_next_task()
Michael spotted that the idle_balance() push down created a task priority problem.
Previously, when we called idle_balance() before pick_next_task() it wasn't a problem when -- because of the rq->lock droppage -- an rt/dl task slipped in.
Similarly for pre_schedule(), rt pre-schedule could have a dl task slip in.
But by pulling it into the pick_next_task() loop, we'll not try a higher task priority again.
Cure this by creating a re-start condition in pick_next_task(); and triggering this from pick_next_task_{rt,fair}().
It also fixes a live-lock where we get stuck in pick_next_task_fair() due to idle_balance() seeing !0 nr_running but there not actually being any fair tasks about.
--- kernel/sched/core.c | 12 ++++++++---- kernel/sched/fair.c | 13 ++++++++++++- kernel/sched/rt.c | 10 +++++++++- kernel/sched/sched.h | 5 +++++ 4 files changed, 34 insertions(+), 6 deletions(-)
kernel/sched/core.c +8 -4
@@ -2586,24 +2586,28 @@ static inline void schedule_debug(struct task_struct *prev)static inline struct task_struct *pick_next_task (struct rq *rq,struct task_struct *prev) { -const struct sched_class *class; +const struct sched_class *class = &fair_sched_class;struct task_struct *p;/* * Optimization: we know that if all tasks are in * the fair class we can call that function directly: */ -if (likely (prev->sched_class == &fair_sched_class && +if (likely (prev->sched_class == class && rq->nr_running == rq->cfs.h_nr_running)) { p = fair_sched_class.pick_next_task (rq, prev); -if (likely (p)) +if (likely (p && p !=RETRY_TASK ))return p; } +again:for_each_class (class) { p = class->pick_next_task (rq, prev); -if (p) +if (p) { +if (unlikely (p ==RETRY_TASK )) + goto again;return p; + } }BUG ();/* the idle class will always have a runnable task */
kernel/sched/fair.c +12 -1
@@ -4686,6 +4686,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev)struct cfs_rq *cfs_rq = &rq->cfs;struct sched_entity *se;struct task_struct *p; +int new_tasks; again: #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -4784,7 +4785,17 @@ simple:return p; idle: -if (idle_balance (rq))/* drops rq->lock */ +/* +* Because idle_balance() releases (and re-acquires) rq->lock, it is +* possible for any higher priority task to appear. In that case we +* must re-start the pick_next_entity() loop. +*/ + new_tasks =idle_balance (rq); + +if (rq->nr_running != rq->cfs.h_nr_running) +return RETRY_TASK ; + +if (new_tasks) goto again;return NULL ;
kernel/sched/rt.c +9 -1
@@ -1360,8 +1360,16 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev)struct task_struct *p;struct rt_rq *rt_rq = &rq->rt; -if (need_pull_rt_task (rq, prev)) +if (need_pull_rt_task (rq, prev)) {pull_rt_task (rq); +/* +* pull_rt_task() can drop (and re-acquire) rq->lock; this +* means a dl task can slip in, in which case we need to +* re-start task selection. +*/ +if (unlikely (rq->dl.dl_nr_running)) +return RETRY_TASK ; + }if (!rt_rq->rt_nr_running)return NULL ;
kernel/sched/sched.h +5 -0
@@ -1091,6 +1091,8 @@ static const u32 prio_to_wmult[40] = {#define DEQUEUE_SLEEP 1 +#define RETRY_TASK ((void *)-1UL) +struct sched_class {const struct sched_class *next;
@@ -1105,6 +1107,9 @@ struct sched_class {* It is the responsibility of the pick_next_task() method that will * return the next task to call put_prev_task() on the @prev task or * something equivalent. +* +* May return RETRY_TASK when it finds a higher prio class has runnable +* tasks. */ struct task_struct * (*pick_next_task) (struct rq *rq,struct task_struct *prev);
Quoted 20 lines · Peter Zijlstra
Subject: sched: Guarantee task priority in
pick_next_task()
From: Peter Zijlstra <[email protected]>
Date: Fri Feb 14 12:25:08 CET 2014Michael spotted that the
idle_balance()push down created a task priority problem.Previously, when we called
idle_balance()beforepick_next_task()it wasn't a problem when -- because of therq->lockdroppage -- an rt/dl task slipped in.Similarly for
pre_schedule(), rt pre-schedule could have a dl task slip in.But by pulling it into the
pick_next_task()loop, we'll not try a higher task priority again.Cure this by creating a re-start condition in
pick_next_task(); and triggering this from pick_next_task_{rt,fair}().
Quoted 6 lines
Fixes 38033c37faab ("sched: Push down pre_schedule() and idle_balance()")Cc Juri Lelli <[email protected]>Cc Ingo Molnar <[email protected]>Cc Steven Rostedt <[email protected]>Reported-by Michael Wang <[email protected]>Signed-off-by Peter Zijlstra <[email protected]>
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
Stack trace is similar to before:
[ 6004.990292] CPU: 20 PID: 26054 Comm: trinity-c58 Not tainted 3.14.0-next-20140409-sasha-00022-g984f7c5-dirty #385 [ 6004.990292] task: ffff880375bb3000 ti: ffff88036058e000 task.ti: ffff88036058e000 [ 6004.990292] RIP: generic_exec_single (kernel/smp.c:91 kernel/smp.c:175) [ 6004.990292] RSP: 0000:ffff88036058f978 EFLAGS: 00000202 [ 6004.990292] RAX: ffff8802b71dec00 RBX: ffff88036058f978 RCX: ffff8802b71decd8 [ 6004.990292] RDX: ffff8802b71d85c0 RSI: ffff88036058f978 RDI: ffff88036058f978 [ 6004.990292] RBP: ffff88036058f9c8 R08: 0000000000000001 R09: ffffffffa70bc580 [ 6004.990292] R10: ffff880375bb3000 R11: 0000000000000000 R12: 000000000000000c [ 6004.990292] R13: 0000000000000001 R14: ffff88036058fa20 R15: ffffffffa121f560 [ 6004.990292] FS: 00007fe993fbd700(0000) GS:ffff880437000000(0000) knlGS:0000000000000000 [ 6004.990292] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 6004.990292] CR2: 00007fffb56b0a18 CR3: 00000003755df000 CR4: 00000000000006a0 [ 6004.990292] DR0: 0000000000695000 DR1: 0000000000695000 DR2: 0000000000000000 [ 6004.990292] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 6004.990292] Stack: [ 6004.990292] ffff88040513da18 ffffffffa121f560 ffff88036058fa20 0000000000000002 [ 6004.990292] 000000000000000c 000000000000000c ffffffffa121f560 ffff88036058fa20 [ 6004.990292] 0000000000000001 ffff880189fe3000 ffff88036058fa08 ffffffffa11ff7b2 [ 6004.990292] Call Trace: [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] smp_call_function_single (kernel/smp.c:234 (discriminator 7)) [ 6004.990292] ? lg_local_lock (kernel/locking/lglock.c:25) [ 6004.990292] stop_two_cpus (kernel/stop_machine.c:297) [ 6004.990292] ? retint_restore_args (arch/x86/kernel/entry_64.S:1040) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __migrate_swap_task (kernel/sched/core.c:1042) [ 6004.990292] migrate_swap (kernel/sched/core.c:1110) [ 6004.990292] task_numa_migrate (kernel/sched/fair.c:1321) [ 6004.990292] ? task_numa_migrate (kernel/sched/fair.c:1227) [ 6004.990292] ? sched_clock_cpu (kernel/sched/clock.c:311) [ 6004.990292] numa_migrate_preferred (kernel/sched/fair.c:1342) [ 6004.990292] task_numa_fault (kernel/sched/fair.c:1796) [ 6004.990292] __handle_mm_fault (mm/memory.c:3812 mm/memory.c:3812 mm/memory.c:3925) [ 6004.990292] ? __const_udelay (arch/x86/lib/delay.c:126) [ 6004.990292] ? __rcu_read_unlock (kernel/rcu/update.c:97) [ 6004.990292] handle_mm_fault (include/linux/memcontrol.h:147 mm/memory.c:3951) [ 6004.990292] __do_page_fault (arch/x86/mm/fault.c:1220) [ 6004.990292] ? vtime_account_user (kernel/sched/cputime.c:687) [ 6004.990292] ? get_parent_ip (kernel/sched/core.c:2472) [ 6004.990292] ? context_tracking_user_exit (include/linux/vtime.h:89 include/linux/jump_label.h:105 include/trace/events/context_tracking.h:47 kernel/context_tracking.c:178) [ 6004.990292] ? preempt_count_sub (kernel/sched/core.c:2527) [ 6004.990292] ? context_tracking_user_exit (kernel/context_tracking.c:182) [ 6004.990292] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 6004.990292] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2638 (discriminator 2)) [ 6004.990292] do_page_fault (arch/x86/mm/fault.c:1272 include/linux/jump_label.h:105 include/linux/context_tracking_state.h:27 include/linux/context_tracking.h:45 arch/x86/mm/fault.c:1273) [ 6004.990292] do_async_page_fault (arch/x86/kernel/kvm.c:263) [ 6004.990292] async_page_fault (arch/x86/kernel/entry_64.S:1496) [ 6004.990292] Code: 44 89 e7 ff 15 70 2d c5 04 45 85 ed 75 0b 31 c0 eb 27 0f 1f 80 00 00 00 00 f6 43 18 01 74 ef 66 2e 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 18 01 75 f8 eb db 66 0f 1f 44 00 00 48 83 c4 28 5b 41 5c
Thanks,
Sasha
On 04/10/2014 11:31 AM, Sasha Levin wrote:
[snip]
Quoted 4 lines
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
A new mechanism has been designed to move the priority checking inside idle_balance(), including Kirill who is the designer ;-)
Regards,
Michael Wang
Quoted 63 lines
Stack trace is similar to before:
[ 6004.990292] CPU: 20 PID: 26054 Comm: trinity-c58 Not tainted 3.14.0-next-20140409-sasha-00022-g984f7c5-dirty #385 [ 6004.990292] task: ffff880375bb3000 ti: ffff88036058e000 task.ti: ffff88036058e000 [ 6004.990292] RIP: generic_exec_single (kernel/smp.c:91 kernel/smp.c:175) [ 6004.990292] RSP: 0000:ffff88036058f978 EFLAGS: 00000202 [ 6004.990292] RAX: ffff8802b71dec00 RBX: ffff88036058f978 RCX: ffff8802b71decd8 [ 6004.990292] RDX: ffff8802b71d85c0 RSI: ffff88036058f978 RDI: ffff88036058f978 [ 6004.990292] RBP: ffff88036058f9c8 R08: 0000000000000001 R09: ffffffffa70bc580 [ 6004.990292] R10: ffff880375bb3000 R11: 0000000000000000 R12: 000000000000000c [ 6004.990292] R13: 0000000000000001 R14: ffff88036058fa20 R15: ffffffffa121f560 [ 6004.990292] FS: 00007fe993fbd700(0000) GS:ffff880437000000(0000) knlGS:0000000000000000 [ 6004.990292] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 6004.990292] CR2: 00007fffb56b0a18 CR3: 00000003755df000 CR4: 00000000000006a0 [ 6004.990292] DR0: 0000000000695000 DR1: 0000000000695000 DR2: 0000000000000000 [ 6004.990292] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 6004.990292] Stack: [ 6004.990292] ffff88040513da18 ffffffffa121f560 ffff88036058fa20 0000000000000002 [ 6004.990292] 000000000000000c 000000000000000c ffffffffa121f560 ffff88036058fa20 [ 6004.990292] 0000000000000001 ffff880189fe3000 ffff88036058fa08 ffffffffa11ff7b2 [ 6004.990292] Call Trace: [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] smp_call_function_single (kernel/smp.c:234 (discriminator 7)) [ 6004.990292] ? lg_local_lock (kernel/locking/lglock.c:25) [ 6004.990292] stop_two_cpus (kernel/stop_machine.c:297) [ 6004.990292] ? retint_restore_args (arch/x86/kernel/entry_64.S:1040) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __migrate_swap_task (kernel/sched/core.c:1042) [ 6004.990292] migrate_swap (kernel/sched/core.c:1110) [ 6004.990292] task_numa_migrate (kernel/sched/fair.c:1321) [ 6004.990292] ? task_numa_migrate (kernel/sched/fair.c:1227) [ 6004.990292] ? sched_clock_cpu (kernel/sched/clock.c:311) [ 6004.990292] numa_migrate_preferred (kernel/sched/fair.c:1342) [ 6004.990292] task_numa_fault (kernel/sched/fair.c:1796) [ 6004.990292] __handle_mm_fault (mm/memory.c:3812 mm/memory.c:3812 mm/memory.c:3925) [ 6004.990292] ? __const_udelay (arch/x86/lib/delay.c:126) [ 6004.990292] ? __rcu_read_unlock (kernel/rcu/update.c:97) [ 6004.990292] handle_mm_fault (include/linux/memcontrol.h:147 mm/memory.c:3951) [ 6004.990292] __do_page_fault (arch/x86/mm/fault.c:1220) [ 6004.990292] ? vtime_account_user (kernel/sched/cputime.c:687) [ 6004.990292] ? get_parent_ip (kernel/sched/core.c:2472) [ 6004.990292] ? context_tracking_user_exit (include/linux/vtime.h:89 include/linux/jump_label.h:105 include/trace/events/context_tracking.h:47 kernel/context_tracking.c:178) [ 6004.990292] ? preempt_count_sub (kernel/sched/core.c:2527) [ 6004.990292] ? context_tracking_user_exit (kernel/context_tracking.c:182) [ 6004.990292] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 6004.990292] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2638 (discriminator 2)) [ 6004.990292] do_page_fault (arch/x86/mm/fault.c:1272 include/linux/jump_label.h:105 include/linux/context_tracking_state.h:27 include/linux/context_tracking.h:45 arch/x86/mm/fault.c:1273) [ 6004.990292] do_async_page_fault (arch/x86/kernel/kvm.c:263) [ 6004.990292] async_page_fault (arch/x86/kernel/entry_64.S:1496) [ 6004.990292] Code: 44 89 e7 ff 15 70 2d c5 04 45 85 ed 75 0b 31 c0 eb 27 0f 1f 80 00 00 00 00 f6 43 18 01 74 ef 66 2e 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 18 01 75 f8 eb db 66 0f 1f 44 00 00 48 83 c4 28 5b 41 5cThanks,
Sasha
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Peter Zijlstra
maintainerQuoted 3 lines · Sasha Levin
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
Weird; we didn't touch anything in the last few weeks :-/
Quoted 44 lines
Stack trace is similar to before:
[ 6004.990292] CPU: 20 PID: 26054 Comm: trinity-c58 Not tainted 3.14.0-next-20140409-sasha-00022-g984f7c5-dirty #385 [ 6004.990292] task: ffff880375bb3000 ti: ffff88036058e000 task.ti: ffff88036058e000 [ 6004.990292] RIP: generic_exec_single (kernel/smp.c:91 kernel/smp.c:175) [ 6004.990292] RSP: 0000:ffff88036058f978 EFLAGS: 00000202 [ 6004.990292] RAX: ffff8802b71dec00 RBX: ffff88036058f978 RCX: ffff8802b71decd8 [ 6004.990292] RDX: ffff8802b71d85c0 RSI: ffff88036058f978 RDI: ffff88036058f978 [ 6004.990292] RBP: ffff88036058f9c8 R08: 0000000000000001 R09: ffffffffa70bc580 [ 6004.990292] R10: ffff880375bb3000 R11: 0000000000000000 R12: 000000000000000c [ 6004.990292] R13: 0000000000000001 R14: ffff88036058fa20 R15: ffffffffa121f560 [ 6004.990292] FS: 00007fe993fbd700(0000) GS:ffff880437000000(0000) knlGS:0000000000000000 [ 6004.990292] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 6004.990292] CR2: 00007fffb56b0a18 CR3: 00000003755df000 CR4: 00000000000006a0 [ 6004.990292] DR0: 0000000000695000 DR1: 0000000000695000 DR2: 0000000000000000 [ 6004.990292] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 6004.990292] Stack: [ 6004.990292] ffff88040513da18 ffffffffa121f560 ffff88036058fa20 0000000000000002 [ 6004.990292] 000000000000000c 000000000000000c ffffffffa121f560 ffff88036058fa20 [ 6004.990292] 0000000000000001 ffff880189fe3000 ffff88036058fa08 ffffffffa11ff7b2 [ 6004.990292] Call Trace: [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] ? cpu_stop_queue_work (kernel/stop_machine.c:227) [ 6004.990292] smp_call_function_single (kernel/smp.c:234 (discriminator 7)) [ 6004.990292] ? lg_local_lock (kernel/locking/lglock.c:25) [ 6004.990292] stop_two_cpus (kernel/stop_machine.c:297) [ 6004.990292] ? retint_restore_args (arch/x86/kernel/entry_64.S:1040) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __stop_cpus (kernel/stop_machine.c:170) [ 6004.990292] ? __migrate_swap_task (kernel/sched/core.c:1042) [ 6004.990292] migrate_swap (kernel/sched/core.c:1110) [ 6004.990292] task_numa_migrate (kernel/sched/fair.c:1321) [ 6004.990292] ? task_numa_migrate (kernel/sched/fair.c:1227) [ 6004.990292] ? sched_clock_cpu (kernel/sched/clock.c:311) [ 6004.990292] numa_migrate_preferred (kernel/sched/fair.c:1342) [ 6004.990292] task_numa_fault (kernel/sched/fair.c:1796) [ 6004.990292] __handle_mm_fault (mm/memory.c:3812 mm/memory.c:3812 mm/memory.c:3925) [ 6004.990292] ? __const_udelay (arch/x86/lib/delay.c:126) [ 6004.990292] ? __rcu_read_unlock (kernel/rcu/update.c:97) [ 6004.990292] handle_mm_fault (include/linux/memcontrol.h:147 mm/memory.c:3951) [ 6004.990292] __do_page_fault (arch/x86/mm/fault.c:1220) [ 6004.990292] ? vtime_account_user (kernel/sched/cputime.c:687) [ 6004.990292] ? get_parent_ip (kernel/sched/core.c:2472) [ 6004.990292] ? context_tracking_user_exit (include/linux/vtime.h:89 include/linux/jump_label.h:105 include/trace/events/context_tracking.h:47 kernel/context_tracking.c:178)
Quoted 8 lines
[ 6004.990292] ? preempt_count_sub (kernel/sched/core.c:2527) [ 6004.990292] ? context_tracking_user_exit (kernel/context_tracking.c:182) [ 6004.990292] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 6004.990292] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2638 (discriminator 2)) [ 6004.990292] do_page_fault (arch/x86/mm/fault.c:1272 include/linux/jump_label.h:105 include/linux/context_tracking_state.h:27 include/linux/context_tracking.h:45 arch/x86/mm/fault.c:1273) [ 6004.990292] do_async_page_fault (arch/x86/kernel/kvm.c:263) [ 6004.990292] async_page_fault (arch/x86/kernel/entry_64.S:1496) [ 6004.990292] Code: 44 89 e7 ff 15 70 2d c5 04 45 85 ed 75 0b 31 c0 eb 27 0f 1f 80 00 00 00 00 f6 43 18 01 74 ef 66 2e 0f 1f 84 00 00 00 00 00 f3 90 <f6> 43 18 01 75 f8 eb db 66 0f 1f 44 00 00 48 83 c4 28 5b 41 5c
This different stack trace format throws your brain...
In any case, do any of the other CPUs do anything interesting?
10.04.2014, 11:00, "Michael wang" <[email protected]>:
Quoted 9 lines · Sasha Levin
[snip]
Quoted 3 lines
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
A new mechanism has been designed to move the priority checking inside
idle_balance(), including Kirill who is the designer ;-)
Not sure, it's connected with my patch. But looks like, we forgot exactly stop class. Maybe this will help?
[PATCH] sched: Checking for stop task appearance when balancing happens
Just do it, like we do for other higher priority classes...
--- kernel/sched/deadline.c | 11 ++++++++++- kernel/sched/fair.c | 3 ++- kernel/sched/rt.c | 7 ++++--- 3 files changed, 16 insertions(+), 5 deletions(-)
kernel/sched/deadline.c +10 -1
@@ -1021,8 +1021,17 @@ struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev) dl_rq = &rq->dl; -if (need_pull_dl_task (rq, prev)) +if (need_pull_dl_task (rq, prev)) {pull_dl_task (rq); +/* +* pull_rt_task() can drop (and re-acquire) rq->lock; this +* means a stop task can slip in, in which case we need to +* re-start task selection. +*/ +if (rq->stop && rq->stop->on_rq) +return RETRY_TASK ; + } +/* * When prev is DL, we may throttle it in put_prev_task(). * So, we update time before we check for dl_nr_running.
kernel/sched/fair.c +2 -1
@@ -6727,7 +6727,8 @@ static int idle_balance(struct rq *this_rq) out:/* Is there a task of a high priority class? */ if (this_rq->nr_running != this_rq->cfs.h_nr_running && - (this_rq->dl.dl_nr_running || + ((this_rq->stop && this_rq->stop->on_rq) || + this_rq->dl.dl_nr_running || (this_rq->rt.rt_nr_running && !rt_rq_throttled (&this_rq->rt)))) pulled_task =-1 ;
kernel/sched/rt.c +4 -3
@@ -1362,10 +1362,11 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev)pull_rt_task (rq);/* * pull_rt_task() can drop (and re-acquire) rq->lock; this -* means a dl task can slip in, in which case we need to -* re-start task selection. +* means a dl or stop task can slip in, in which case we need +* to re-start task selection. */ -if (unlikely (rq->dl.dl_nr_running)) +if (unlikely ((rq->stop && rq->stop->on_rq) || + rq->dl.dl_nr_running))return RETRY_TASK ; }
Quoted 10 lines · Kirill Tkhai
10.04.2014, 11:00, "Michael wang" <[email protected]>:
Quoted 9 lines
Quoted 9 lines · Sasha Levin
[snip]
Quoted 3 lines
Quoted 3 lines
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
A new mechanism has been designed to move the priority checking inside
idle_balance(), including Kirill who is the designer ;-)
Quoted 8 lines
Not sure, it's connected with my patch. But looks like, we forgot exactly stop class. Maybe this will help?
[PATCH] sched: Checking for stop task appearance when balancing happens
Just do it, like we do for other higher priority classes...
Signed-off-by Kirill Tkhai <[email protected]>
I've been running with this patch for the last two days and the hang seems to be gone.
I'll leave it running on the weekend and will update again on Monday.
Thanks,
Sasha
11.04.2014, 18:33, "Sasha Levin" <[email protected]>:
Quoted 15 lines · Kirill Tkhai
Quoted 14 lines
10.04.2014, 11:00, "Michael wang" <[email protected]>:
Quoted 7 lines
Quoted 7 lines · Sasha Levin
[snip]
Quoted 3 lines
Quoted 3 lines
I'd like to re-open this issue. It seems that something broke and I'm now seeing the same issues that have gone away 2 months with this patch again.
A new mechanism has been designed to move the priority checking inside
idle_balance(), including Kirill who is the designer ;-)Not sure, it's connected with my patch. But looks like, we forgot exactly stop class. Maybe this will help?
[PATCH] sched: Checking for stop task appearance when balancing happens
Quoted 8 lines
Quoted 3 lines
Just do it, like we do for other higher priority classes...
Signed-off-by Kirill Tkhai <[email protected]>I've been running with this patch for the last two days and the hang seems to be gone.
I'll leave it running on the weekend and will update again on Monday.
Thanks for testing.
Sorry for I killed your linuxes by this lost need_resched().
Quoted 3 lines
Thanks,
Sasha
tip-bot for Kirill Tkhai
merge bot[tip:sched/urgent] sched: Check for stop task appearance when balancing happens
In reply to Kirill Tkhaisched: Check for stop task appearance when balancing happens
Commit-ID: a1d9a3231eac4117cadaf4b6bba5b2902c15a33e Gitweb: http://git.kernel.org/tip/a1d9a3231eac4117cadaf4b6bba5b2902c15a33e Author: Kirill Tkhai <[email protected]> AuthorDate: Thu, 10 Apr 2014 17:38:36 +0400 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 17 Apr 2014 13:39:51 +0200
sched: Check for stop task appearance when balancing happens
We need to do it like we do for the other higher priority classes..
--- kernel/sched/deadline.c | 11 ++++++++++- kernel/sched/fair.c | 3 ++- kernel/sched/rt.c | 7 ++++--- 3 files changed, 16 insertions(+), 5 deletions(-)
kernel/sched/deadline.c +10 -1
@@ -1021,8 +1021,17 @@ struct task_struct *pick_next_task_dl(struct rq *rq, struct task_struct *prev) dl_rq = &rq->dl; -if (need_pull_dl_task (rq, prev)) +if (need_pull_dl_task (rq, prev)) {pull_dl_task (rq); +/* +* pull_rt_task() can drop (and re-acquire) rq->lock; this +* means a stop task can slip in, in which case we need to +* re-start task selection. +*/ +if (rq->stop && rq->stop->on_rq) +return RETRY_TASK ; + } +/* * When prev is DL, we may throttle it in put_prev_task(). * So, we update time before we check for dl_nr_running.
kernel/sched/fair.c +2 -1
@@ -6728,7 +6728,8 @@ static int idle_balance(struct rq *this_rq) out:/* Is there a task of a high priority class? */ if (this_rq->nr_running != this_rq->cfs.h_nr_running && - (this_rq->dl.dl_nr_running || + ((this_rq->stop && this_rq->stop->on_rq) || + this_rq->dl.dl_nr_running || (this_rq->rt.rt_nr_running && !rt_rq_throttled (&this_rq->rt)))) pulled_task =-1 ;
kernel/sched/rt.c +4 -3
@@ -1362,10 +1362,11 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev)pull_rt_task (rq);/* * pull_rt_task() can drop (and re-acquire) rq->lock; this -* means a dl task can slip in, in which case we need to -* re-start task selection. +* means a dl or stop task can slip in, in which case we need +* to re-start task selection. */ -if (unlikely (rq->dl.dl_nr_running)) +if (unlikely ((rq->stop && rq->stop->on_rq) || + rq->dl.dl_nr_running))return RETRY_TASK ; }
Quoted 20 lines · Kirill Tkhai
11.04.2014, 18:33, "Sasha Levin" <[email protected]>:
Quoted 19 lines
Quoted 19 lines · Kirill Tkhai
Quoted 17 lines
Quoted 17 lines
10.04.2014, 11:00, "Michael wang" <[email protected]>:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines · Sasha Levin
[snip]
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
I'd like to re-open this issue. It seems that something broke and I'm
now seeing the same issues that have gone away 2 months with this patch
again.A new mechanism has been designed to move the priority checking inside
idle_balance(), including Kirill who is the designer ;-)Not sure, it's connected with my patch. But looks like, we forgot exactly stop class. Maybe this will help?
[PATCH] sched: Checking for stop task appearance when balancing happens
Just do it, like we do for other higher priority classes...
Signed-off-by Kirill Tkhai <[email protected]>
Quoted 7 lines
Quoted 4 lines
Quoted 4 lines
I've been running with this patch for the last two days and the hang seems to be gone.
I'll leave it running on the weekend and will update again on Monday.
Thanks for testing.
Sorry for I killed your linuxes by this lost
need_resched().
So I'm going to revive this thread again:
[ 3912.086567] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [migration/4:194] [ 3912.089060] Modules linked in: [ 3912.090438] irq event stamp: 585772 [ 3912.090605] hardirqs last enabled at (585771): restore_args (arch/x86/kernel/entry_64.S:1033) [ 3912.090605] hardirqs last disabled at (585772): trace_apic_timer_interrupt (arch/x86/kernel/entry_64.S:1225) [ 3912.090605] softirqs last enabled at (585770): __do_softirq (arch/x86/include/asm/preempt.h:22 kernel/softirq.c:296) [ 3912.090605] softirqs last disabled at (585763): irq_exit (kernel/softirq.c:346 kernel/softirq.c:387) [ 3912.090605] CPU: 4 PID: 194 Comm: migration/4 Tainted: G W 3.15.0-rc5-next-20140512-sasha-00019-ga20bc00-dirty #456 [ 3912.090605] task: ffff8801445a8000 ti: ffff880144596000 task.ti: ffff880144596000 [ 3912.090605] RIP: multi_cpu_stop (kernel/stop_machine.c:210) [ 3912.090605] RSP: 0000:ffff880144597cc8 EFLAGS: 00000293 [ 3912.090605] RAX: 0000000000000001 RBX: ffffffffa458d3fb RCX: 0000000000000034 [ 3912.090605] RDX: ffff8801445a8000 RSI: 0000000000000034 RDI: 0000000020000000 [ 3912.090605] RBP: ffff880144597d08 R08: 0000000000000001 R09: 0000000000000000 [ 3912.090605] R10: ffffffffa4638060 R11: 0000000000000001 R12: ffff880144597c38 [ 3912.090605] R13: 0000000000000001 R14: ffff880144596000 R15: ffff8801445a8000 [ 3912.090605] FS: 0000000000000000(0000) GS:ffff880144c00000(0000) knlGS:0000000000000000 [ 3912.090605] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 3912.090605] CR2: 000000000258aff8 CR3: 000000002602c000 CR4: 00000000000006a0 [ 3912.090605] DR0: 00000000006df000 DR1: 00000000006df000 DR2: 0000000000000000 [ 3912.090605] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 3912.090605] Stack: [ 3912.090605] ffff880144597ce8 0000000000000286 ffff880144597ce8 ffff88015666fa40 [ 3912.090605] ffff880144dd0a00 ffff88015666fa18 ffffffffa1227180 ffff880144dd0a50 [ 3912.090605] ffff880144597dd8 ffffffffa1226e42 ffff880144597d28 ffffffffa11c4aae [ 3912.090605] Call Trace: [ 3912.090605] ? queue_stop_cpus_work (kernel/stop_machine.c:171) [ 3912.090605] cpu_stopper_thread (kernel/stop_machine.c:498) [ 3912.090605] ? put_lock_stats.isra.12 (arch/x86/include/asm/preempt.h:98 kernel/locking/lockdep.c:254) [ 3912.090605] ? _raw_spin_unlock_irqrestore (arch/x86/include/asm/paravirt.h:809 include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 3912.090605] ? trace_hardirqs_on_caller (kernel/locking/lockdep.c:2557 kernel/locking/lockdep.c:2599) [ 3912.090605] smpboot_thread_fn (kernel/smpboot.c:160) [ 3912.090605] ? __smpboot_create_thread (kernel/smpboot.c:105) [ 3912.090605] kthread (kernel/kthread.c:210) [ 3912.090605] ? complete (kernel/sched/completion.c:35) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] ret_from_fork (arch/x86/kernel/entry_64.S:553) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] Code: 09 06 0f 1f 80 00 00 00 00 4d 0f a3 65 00 45 19 e4 45 85 e4 41 0f 95 c6 e8 bc 10 90 00 41 89 c4 45 31 ed 31 c0 0f 1f 40 00 f3 90 <45> 84 f6 75 11 44 8b 7b 20 41 39 c7 75 20 eb 67 66 0f 1f 44 00 [ 3912.090605] Kernel panic - not syncing: softlockup: hung tasks [ 3912.090605] CPU: 4 PID: 194 Comm: migration/4 Tainted: G W 3.15.0-rc5-next-20140512-sasha-00019-ga20bc00-dirty #456 [ 3912.090605] ffff8801445a8000 ffff880144c03dd8 ffffffffa453e1ec 0000000000000001 [ 3912.090605] ffffffffa56e9ed0 ffff880144c03e58 ffffffffa452fd6a 0000000000000001 [ 3912.090605] 0000000000000008 ffff880144c03e68 ffff880144c03e08 0000000000000001 [ 3912.090605] Call Trace: [ 3912.090605] <IRQ> dump_stack (lib/dump_stack.c:52) [ 3912.090605] panic (kernel/panic.c:119) [ 3912.090605] ? _raw_spin_unlock_irqrestore (include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] watchdog_timer_fn (kernel/watchdog.c:372) [ 3912.090605] __run_hrtimer (kernel/hrtimer.c:1268 (discriminator 2)) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] ? watchdog (kernel/hung_task.c:191 kernel/hung_task.c:232) [ 3912.090605] hrtimer_interrupt (kernel/hrtimer.c:1915) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] local_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:921) [ 3912.090605] smp_trace_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:964 include/linux/jump_label.h:105 arch/x86/include/asm/trace/irq_vectors.h:45 arch/x86/kernel/apic/apic.c:965) [ 3912.090605] trace_apic_timer_interrupt (arch/x86/kernel/entry_64.S:1225) [ 3912.090605] <EOI> ? retint_restore_args (arch/x86/kernel/entry_64.S:1033) [ 3912.090605] ? multi_cpu_stop (kernel/stop_machine.c:210) [ 3912.090605] ? multi_cpu_stop (include/linux/bitmap.h:280 include/linux/cpumask.h:461 kernel/stop_machine.c:189) [ 3912.090605] ? queue_stop_cpus_work (kernel/stop_machine.c:171) [ 3912.090605] cpu_stopper_thread (kernel/stop_machine.c:498) [ 3912.090605] ? put_lock_stats.isra.12 (arch/x86/include/asm/preempt.h:98 kernel/locking/lockdep.c:254) [ 3912.090605] ? _raw_spin_unlock_irqrestore (arch/x86/include/asm/paravirt.h:809 include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 3912.090605] ? trace_hardirqs_on_caller (kernel/locking/lockdep.c:2557 kernel/locking/lockdep.c:2599) [ 3912.090605] smpboot_thread_fn (kernel/smpboot.c:160) [ 3912.090605] ? __smpboot_create_thread (kernel/smpboot.c:105) [ 3912.090605] kthread (kernel/kthread.c:210) [ 3912.090605] ? complete (kernel/sched/completion.c:35) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] ret_from_fork (arch/x86/kernel/entry_64.S:553) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] Dumping ftrace buffer: [ 3912.090605] (ftrace buffer empty)
Nothing else is printed past this point.
Thanks,
Sasha
12.05.2014, 22:49, "Sasha Levin" <[email protected]>:
Quoted 23 lines · Kirill Tkhai
Quoted 22 lines
11.04.2014, 18:33, "Sasha Levin" <[email protected]>:
Quoted 20 lines
Quoted 20 lines · Kirill Tkhai
Quoted 8 lines
Quoted 8 lines
10.04.2014, 11:00, "Michael wang" <[email protected]>:
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines · Sasha Levin
[snip]
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
I'd like to re-open this issue. It seems that something broke and I'm
now seeing the same issues that have gone away 2 months with this patch
again.A new mechanism has been designed to move the priority checking inside
idle_balance(), including Kirill who is the designer ;-)Quoted 8 lines
Quoted 8 lines
Not sure, it's connected with my patch. But looks like, we forgot exactly stop class. Maybe this will help?
[PATCH] sched: Checking for stop task appearance when balancing happens
Just do it, like we do for other higher priority classes...
Signed-off-by Kirill Tkhai <[email protected]>I've been running with this patch for the last two days and the hang seems to be gone.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
I'll leave it running on the weekend and will update again on Monday.
Thanks for testing.
Sorry for I killed your linuxes by this lost
need_resched().So I'm going to revive this thread again:
Sadly, kernel parameter "softlockup_panic" prevents stack dumping on other cpus..
Just as an idea.
Peter, do we have to queue stop works orderly?
Is there is not a possibility, when two pair of works queued different on different cpus?
kernel/stop_machine.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
kernel/stop_machine.c +8 -2
@@ -250,8 +250,14 @@ struct irq_cpu_stop_queue_work_info {static void irq_cpu_stop_queue_work (void *arg) {struct irq_cpu_stop_queue_work_info *info = arg; -cpu_stop_queue_work (info->cpu1, info->work1); -cpu_stop_queue_work (info->cpu2, info->work2); + +if (info->cpu1 < info->cpu2) { +cpu_stop_queue_work (info->cpu1, info->work1); +cpu_stop_queue_work (info->cpu2, info->work2); + }else { +cpu_stop_queue_work (info->cpu2, info->work2); +cpu_stop_queue_work (info->cpu1, info->work1); + } }/**
Quoted 82 lines
[ 3912.086567] NMI watchdog: BUG: soft lockup - CPU#4 stuck for 22s! [migration/4:194] [ 3912.089060] Modules linked in: [ 3912.090438] irq event stamp: 585772 [ 3912.090605] hardirqs last enabled at (585771): restore_args (arch/x86/kernel/entry_64.S:1033) [ 3912.090605] hardirqs last disabled at (585772): trace_apic_timer_interrupt (arch/x86/kernel/entry_64.S:1225) [ 3912.090605] softirqs last enabled at (585770): __do_softirq (arch/x86/include/asm/preempt.h:22 kernel/softirq.c:296) [ 3912.090605] softirqs last disabled at (585763): irq_exit (kernel/softirq.c:346 kernel/softirq.c:387) [ 3912.090605] CPU: 4 PID: 194 Comm: migration/4 Tainted: G W 3.15.0-rc5-next-20140512-sasha-00019-ga20bc00-dirty #456 [ 3912.090605] task: ffff8801445a8000 ti: ffff880144596000 task.ti: ffff880144596000 [ 3912.090605] RIP: multi_cpu_stop (kernel/stop_machine.c:210) [ 3912.090605] RSP: 0000:ffff880144597cc8 EFLAGS: 00000293 [ 3912.090605] RAX: 0000000000000001 RBX: ffffffffa458d3fb RCX: 0000000000000034 [ 3912.090605] RDX: ffff8801445a8000 RSI: 0000000000000034 RDI: 0000000020000000 [ 3912.090605] RBP: ffff880144597d08 R08: 0000000000000001 R09: 0000000000000000 [ 3912.090605] R10: ffffffffa4638060 R11: 0000000000000001 R12: ffff880144597c38 [ 3912.090605] R13: 0000000000000001 R14: ffff880144596000 R15: ffff8801445a8000 [ 3912.090605] FS: 0000000000000000(0000) GS:ffff880144c00000(0000) knlGS:0000000000000000 [ 3912.090605] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 3912.090605] CR2: 000000000258aff8 CR3: 000000002602c000 CR4: 00000000000006a0 [ 3912.090605] DR0: 00000000006df000 DR1: 00000000006df000 DR2: 0000000000000000 [ 3912.090605] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600 [ 3912.090605] Stack: [ 3912.090605] ffff880144597ce8 0000000000000286 ffff880144597ce8 ffff88015666fa40 [ 3912.090605] ffff880144dd0a00 ffff88015666fa18 ffffffffa1227180 ffff880144dd0a50 [ 3912.090605] ffff880144597dd8 ffffffffa1226e42 ffff880144597d28 ffffffffa11c4aae [ 3912.090605] Call Trace: [ 3912.090605] ? queue_stop_cpus_work (kernel/stop_machine.c:171) [ 3912.090605] cpu_stopper_thread (kernel/stop_machine.c:498) [ 3912.090605] ? put_lock_stats.isra.12 (arch/x86/include/asm/preempt.h:98 kernel/locking/lockdep.c:254) [ 3912.090605] ? _raw_spin_unlock_irqrestore (arch/x86/include/asm/paravirt.h:809 include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 3912.090605] ? trace_hardirqs_on_caller (kernel/locking/lockdep.c:2557 kernel/locking/lockdep.c:2599) [ 3912.090605] smpboot_thread_fn (kernel/smpboot.c:160) [ 3912.090605] ? __smpboot_create_thread (kernel/smpboot.c:105) [ 3912.090605] kthread (kernel/kthread.c:210) [ 3912.090605] ? complete (kernel/sched/completion.c:35) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] ret_from_fork (arch/x86/kernel/entry_64.S:553) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] Code: 09 06 0f 1f 80 00 00 00 00 4d 0f a3 65 00 45 19 e4 45 85 e4 41 0f 95 c6 e8 bc 10 90 00 41 89 c4 45 31 ed 31 c0 0f 1f 40 00 f3 90 <45> 84 f6 75 11 44 8b 7b 20 41 39 c7 75 20 eb 67 66 0f 1f 44 00 [ 3912.090605] Kernel panic - not syncing: softlockup: hung tasks [ 3912.090605] CPU: 4 PID: 194 Comm: migration/4 Tainted: G W 3.15.0-rc5-next-20140512-sasha-00019-ga20bc00-dirty #456 [ 3912.090605] ffff8801445a8000 ffff880144c03dd8 ffffffffa453e1ec 0000000000000001 [ 3912.090605] ffffffffa56e9ed0 ffff880144c03e58 ffffffffa452fd6a 0000000000000001 [ 3912.090605] 0000000000000008 ffff880144c03e68 ffff880144c03e08 0000000000000001 [ 3912.090605] Call Trace: [ 3912.090605] <IRQ> dump_stack (lib/dump_stack.c:52) [ 3912.090605] panic (kernel/panic.c:119) [ 3912.090605] ? _raw_spin_unlock_irqrestore (include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] watchdog_timer_fn (kernel/watchdog.c:372) [ 3912.090605] __run_hrtimer (kernel/hrtimer.c:1268 (discriminator 2)) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] ? watchdog (kernel/hung_task.c:191 kernel/hung_task.c:232) [ 3912.090605] hrtimer_interrupt (kernel/hrtimer.c:1915) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] ? queue_stop_cpus_work (include/linux/cpumask.h:108 include/linux/cpumask.h:174 kernel/stop_machine.c:375) [ 3912.090605] local_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:921) [ 3912.090605] smp_trace_apic_timer_interrupt (arch/x86/kernel/apic/apic.c:964 include/linux/jump_label.h:105 arch/x86/include/asm/trace/irq_vectors.h:45 arch/x86/kernel/apic/apic.c:965) [ 3912.090605] trace_apic_timer_interrupt (arch/x86/kernel/entry_64.S:1225) [ 3912.090605] <EOI> ? retint_restore_args (arch/x86/kernel/entry_64.S:1033) [ 3912.090605] ? multi_cpu_stop (kernel/stop_machine.c:210) [ 3912.090605] ? multi_cpu_stop (include/linux/bitmap.h:280 include/linux/cpumask.h:461 kernel/stop_machine.c:189) [ 3912.090605] ? queue_stop_cpus_work (kernel/stop_machine.c:171) [ 3912.090605] cpu_stopper_thread (kernel/stop_machine.c:498) [ 3912.090605] ? put_lock_stats.isra.12 (arch/x86/include/asm/preempt.h:98 kernel/locking/lockdep.c:254) [ 3912.090605] ? _raw_spin_unlock_irqrestore (arch/x86/include/asm/paravirt.h:809 include/linux/spinlock_api_smp.h:160 kernel/locking/spinlock.c:191) [ 3912.090605] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63) [ 3912.090605] ? trace_hardirqs_on_caller (kernel/locking/lockdep.c:2557 kernel/locking/lockdep.c:2599) [ 3912.090605] smpboot_thread_fn (kernel/smpboot.c:160) [ 3912.090605] ? __smpboot_create_thread (kernel/smpboot.c:105) [ 3912.090605] kthread (kernel/kthread.c:210) [ 3912.090605] ? complete (kernel/sched/completion.c:35) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] ret_from_fork (arch/x86/kernel/entry_64.S:553) [ 3912.090605] ? kthread_create_on_node (kernel/kthread.c:176) [ 3912.090605] Dumping ftrace buffer: [ 3912.090605] (ftrace buffer empty)Nothing else is printed past this point.
Thanks,
Sasha
Peter Zijlstra
maintainerQuoted 17 lines · Kirill Tkhai
Peter, do we have to queue stop works orderly?
Is there is not a possibility, when two pair of works queued different on different cpus?
kernel/stop_machine.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)kernel/stop_machine.c +0
-1@@ -250,8 +250,14 @@ struct irq_cpu_stop_queue_work_info {static void irq_cpu_stop_queue_work (void *arg) {struct irq_cpu_stop_queue_work_info *info = arg; -cpu_stop_queue_work (info->cpu1, info->work1);
kernel/stop_machine.c struct irq_cpu_stop_queue_work_info { · line 261
Quoted 10 lines
kernel/stop_machine.c +8
-1-cpu_stop_queue_work (info->cpu2, info->work2); + +if (info->cpu1 < info->cpu2) { +cpu_stop_queue_work (info->cpu1, info->work1); +cpu_stop_queue_work (info->cpu2, info->work2); + }else { +cpu_stop_queue_work (info->cpu2, info->work2); +cpu_stop_queue_work (info->cpu1, info->work1); + } }
I'm not sure, we already send the IPI to the first cpu of the pair, so supposing we have 4 cpus, and get 4 pairs like:
0,1 1,2 2,3 3,0
That would result in IPIs to 0, 1, 2, and 0 again, and since the IPI function is serialized I don't immediately see a way for this to deadlock.
14.05.2014, 14:14, "Peter Zijlstra" <[email protected]>:
Quoted 29 lines · Kirill Tkhai
Quoted 19 lines
Peter, do we have to queue stop works orderly?
Is there is not a possibility, when two pair of works queued different on different cpus?
kernel/stop_machine.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)diff --git
a/kernel/stop_machine.cb/kernel/stop_machine.cindex b6b67ec..29e221b 100644
---a/kernel/stop_machine.c
+++b/kernel/stop_machine.c
@@ -250,8 +250,14 @@ structirq_cpu_stop_queue_work_info{ static voidirq_cpu_stop_queue_work(void *arg)
{
structirq_cpu_stop_queue_work_info*info = arg;
-cpu_stop_queue_work(info->cpu1, info->work1);
-cpu_stop_queue_work(info->cpu2, info->work2);
+Quoted 8 lines
+ if (
info->cpu1<info->cpu2) {
+cpu_stop_queue_work(info->cpu1, info->work1);
+cpu_stop_queue_work(info->cpu2, info->work2);
+ } else {
+cpu_stop_queue_work(info->cpu2, info->work2);
+cpu_stop_queue_work(info->cpu1, info->work1);
+ }
}
Quoted 8 lines
I'm not sure, we already send the IPI to the first cpu of the pair, so supposing we have 4 cpus, and get 4 pairs like:
0,1 1,2 2,3 3,0
That would result in IPIs to 0, 1, 2, and 0 again, and since the IPI function is serialized I don't immediately see a way for this to deadlock.
It's about stop_two_cpus(), I have a distrust about other users of stop task:
queue_stop_cpus_work() queues work consequentially:
0 1 2 4
stop_two_cpus() may queue:
1 0
Looks like, stop thread on 0th and on 1th are waiting for wrong works.
Peter Zijlstra
maintainerQuoted 45 lines · Kirill Tkhai
14.05.2014, 14:14, "Peter Zijlstra" <[email protected]>:
Quoted 29 lines · Kirill Tkhai
Quoted 19 lines
Peter, do we have to queue stop works orderly?
Is there is not a possibility, when two pair of works queued different on different cpus?
kernel/stop_machine.c | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)diff --git
a/kernel/stop_machine.cb/kernel/stop_machine.cindex b6b67ec..29e221b 100644
---a/kernel/stop_machine.c
+++b/kernel/stop_machine.c
@@ -250,8 +250,14 @@ structirq_cpu_stop_queue_work_info{ static voidirq_cpu_stop_queue_work(void *arg)
{
structirq_cpu_stop_queue_work_info*info = arg;
-cpu_stop_queue_work(info->cpu1, info->work1);
-cpu_stop_queue_work(info->cpu2, info->work2);
+Quoted 8 lines
+ if (
info->cpu1<info->cpu2) {
+cpu_stop_queue_work(info->cpu1, info->work1);
+cpu_stop_queue_work(info->cpu2, info->work2);
+ } else {
+cpu_stop_queue_work(info->cpu2, info->work2);
+cpu_stop_queue_work(info->cpu1, info->work1);
+ }
}Quoted 8 lines
I'm not sure, we already send the IPI to the first cpu of the pair, so supposing we have 4 cpus, and get 4 pairs like:
0,1 1,2 2,3 3,0
That would result in IPIs to 0, 1, 2, and 0 again, and since the IPI function is serialized I don't immediately see a way for this to deadlock.
It's about
stop_two_cpus(), I have a distrust about other users of stop task:
queue_stop_cpus_work()queues work consequentially:
Quoted 7 lines
0 1 2 4
stop_two_cpus()may queue:1 0
Looks like, stop thread on 0th and on 1th are waiting for wrong works.
so we serialize stop_cpus_work() vs stop_two_cpus() with an l/g lock.
Ah, but stop_cpus_work() only holds the global lock over queueing, it doesn't wait for completion, that might indeed cause a problem.
Also, since its two different cpus queueing, the ordered queue doesn't really matter, you can still interleave the all and two sets and get into this state.
Peter Zijlstra
maintainerQuoted 4 lines · Peter Zijlstra
so we serialize
stop_cpus_work()vsstop_two_cpus()with an l/g lock.Ah, but
stop_cpus_work()only holds the global lock over queueing, it doesn't wait for completion, that might indeed cause a problem.
Hmm, is this so? If the stop_cpus_work() queueing is complete, the stop_two_cpus() queueing must always happen _after_ it.
Therefore the stopper threads should also observe them in that order, right? /me goes check..
Hmm, cpu_stop_queue_work() does list_add_tail(), does that make it a LIFO?
Bah, now I've confused myself again.
Quoted 3 lines
Also, since its two different cpus queueing, the ordered queue doesn't really matter, you can still interleave the all and two sets and get into this state.
Anyway, before I stared getting self doubts, I wrote the below, now I'm not sure its at all relevant but here goes.
---
Yeah, sorry for not also providing a solution, extending the lg_global lock over the entire sqeuence won't work for it will disable preemption, which means we need something a little smarter.
Now, I still have the hotplug lock rewrite laying about, now Linus had an opinion on that, but we could adapt percpu-rwsem.c to use the same logic.
At that point, I think it would be possible to use the percpu-rwsem for this, use the write lock for the global stop all thing, and the read side for the two-cpu stopper.
Peter, Sasha, coming back to this…
Not that this is happening frequently or I can easily reproduce, but…
Quoted 56 lines · Peter Zijlstra
Quoted 45 lines · Kirill Tkhai
14.05.2014, 14:14, "Peter Zijlstra" <[email protected]>:
Quoted 32 lines · Kirill Tkhai
Quoted 17 lines
Peter, do we have to queue stop works orderly?
Is there is not a possibility, when two pair of works queued different on different cpus?
kernel/stop_machine.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)kernel/stop_machine.c +0
-1@@ -250,8 +250,14 @@ struct irq_cpu_stop_queue_work_info {static void irq_cpu_stop_queue_work (void *arg) {struct irq_cpu_stop_queue_work_info *info = arg; -cpu_stop_queue_work (info->cpu1, info->work1);kernel/stop_machine.c struct irq_cpu_stop_queue_work_info { · line 261
Quoted 10 lines
kernel/stop_machine.c +8
-1-cpu_stop_queue_work (info->cpu2, info->work2); + +if (info->cpu1 < info->cpu2) { +cpu_stop_queue_work (info->cpu1, info->work1); +cpu_stop_queue_work (info->cpu2, info->work2); + }else { +cpu_stop_queue_work (info->cpu2, info->work2); +cpu_stop_queue_work (info->cpu1, info->work1); + } }I'm not sure, we already send the IPI to the first cpu of the pair, so supposing we have 4 cpus, and get 4 pairs like:
Quoted 5 lines
0,1 1,2 2,3 3,0
That would result in IPIs to 0, 1, 2, and 0 again, and since the IPI function is serialized I don't immediately see a way for this to deadlock.
It's about
stop_two_cpus(), I have a distrust about other users of stop task:
queue_stop_cpus_work()queues work consequentially:Quoted 7 lines
0 1 2 4
stop_two_cpus()may queue:1 0
Looks like, stop thread on 0th and on 1th are waiting for wrong works.
Quoted 8 lines
so we serialize
stop_cpus_work()vsstop_two_cpus()with an l/g lock.Ah, but
stop_cpus_work()only holds the global lock over queueing, it doesn't wait for completion, that might indeed cause a problem.Also, since its two different cpus queueing, the ordered queue doesn't really matter, you can still interleave the all and two sets and get into this state.
Do you think __stop_cpus->queue_stop_cpus_work() & stop_two_cpus might be stepping into each other because of this global lock being on held on queuing only (and not completion) ?
In the past I described to Sasha the follow scenario from one of my 3.13 kernels:
Quoted 80 lines
->
multi_cpu_stop-> do { } while (curstate !=MULTI_STOP_EXIT);In my case, curstate is WAY different from enum containing
MULTI_STOP_EXIT(4).Register totally messed up (probably after
cpu_relax(), right where you were trapped -> after the pause instruction).my case:
PID: 118 TASK:
ffff883fd28ec7d0CPU: 9 COMMAND: "migration/9"
...
[exception RIP:multi_cpu_stop+0x64]RIP: ffffffff810f5944 RSP: ffff883fd2907d98 RFLAGS: 00000246 RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000246 RDX: ffff883fd2907d98 RSI: 0000000000000000 RDI: 0000000000000001 RBP: ffffffff810f5944 R8: ffffffff810f5944 R9: 0000000000000000 R10: ffff883fd2907d98 R11: 0000000000000246 R12: ffffffffffffffff R13: ffff883f55d01b48 R14: 0000000000000000 R15: 0000000000000001 ORIG_RAX: 0000000000000001 CS: 0010 SS: 0000--- <NMI exception stack> ---
#4 [ffff883fd2907d98]multi_cpu_stop+0x64atffffffff810f5944208 } while (curstate != MULTI_STOP_EXIT); ---> RIP RIP 0xffffffff810f5944 <+100>: cmp $0x4,%edx ---> CHECKING FOR MULTI_STOP_EXITRDX:
ffff883fd2907d98-> does not make any sense###
If i'm reading this right,
"""
CPU 05 - PID 14990
do_numa_pagetask_numa_faultnuma_migrate_preferredtask_numa_migratemigrate_swap(curr: 14990, task: 14996)stop_two_cpus(cpu1=05(14996), cpu2=00(14990))wait_for_completion14990 - CPU05
14996 - CPU00
stop_two_cpus:
multi_stop_data(msdata->state=MULTI_STOP_PREPARE)smp_call_function_single(min=cpu2=00,irq_cpu_stop_queue_work, wait=1)smp_call_function_single (ran on lowest CPU, 00 for this case) irq_cpu_stop_queue_work cpu_stop_queue_work(cpu1=05(14996)) # add work (multi_cpu_stop) to cpu 05 cpu_stopper queue cpu_stop_queue_work(cpu2=00(14990)) # add work (multi_cpu_stop) to cpu 00 cpu_stopper queue
wait_for_completion()--> HERE
"""in my case, checking task structs for tasks scheduled when "
waiting_for_completion()":PID 14990 CPU 05 -> PID 14996 CPU 00
PID 14991 CPU 30 -> PID 14998 CPU 01
PID 14992 CPU 30 -> PID 14998 CPU 01
PID 14996 CPU 00 -> PID 14992 CPU 30
PID 14998 CPU 01 -> PID 14990 CPU 05AND
102 2 6 ffff881fd2ea97f0 RU 0.0 0 0 [migration/6] 118 2 9 ffff883fd28ec7d0 RU 0.0 0 0 [migration/9] 143 2 14 ffff883fd29d47d0 RU 0.0 0 0 [migration/14] 148 2 15 ffff883fd29fc7d0 RU 0.0 0 0 [migration/15] 153 2 16 ffff881fd2f517f0 RU 0.0 0 0 [migration/16]THEN
I am still waiting for 5
cpu_stopper_thread->multi_cpu_stopjust scheduled (probably in the per cpu's queue of cpus 0,1,5,30), not running yet.
Quoted 5 lines
AND
I don't have any "
wait_for_completion" for those "OLDER" migration threads (6, 9, 14, 15 and 16). Probablywait_for_completionsignalled done.completion before racing.
And following this thread’s discussion, and commits bellow:
commit a1d9a3231eac4117cadaf4b6bba5b2902c15a33e
Author: Kirill Tkhai <[email protected]> Date: Thu Apr 10 17:38:36 2014 +0400
sched: Check for stop task appearance when balancing happens
commit 37e117c07b89194aae7062bc63bde1104c03db02
Author: Peter Zijlstra <[email protected]> Date: Fri Feb 14 12:25:08 2014 +0100
sched: Guarantee task priority in pick_next_task()
commit 38033c37faab850ed5d33bb675c4de6c66be84d8
Author: Peter Zijlstra <[email protected]> Date: Thu Jan 23 20:32:21 2014 +0100
sched: Push down pre_schedule() and idle_balance()
commit 606dba2e289446600a0b68422ed2019af5355c12
Author: Peter Zijlstra <[email protected]> Date: Sat Feb 11 06:05:00 2012 +0100
sched: Push put_prev_task() into pick_next_task()
3.13 kernel still had old logic (before 3.15) - no RETRY_TASK, idle_balance() before pick_next_task(), no deadline scheduler yet - so commit “a1d9a32” does not play a role into this panic. I’m causing ~ 150 stop_two_cpus calls / sec, for task migration, in a 32 fake numa environment, and I am NOT able to reproduce this lockup but, still, the dump is says it is there :\. For 3.13 series this lockup was seen once, no info on other versions.
Any thoughts ?
Thank you
-Rafael Tinoco
Peter Zijlstra
maintainerQuoted 1 line · Rafael David Tinoco
Any thoughts ?
This recently came up again and I proposed the below. Reposting because the original had a silly compile fail.
---
Subject: stop_machine: Fix deadlock between multiple stop_two_cpus()
From: Peter Zijlstra <[email protected]>
Date: Fri, 5 Jun 2015 17:30:23 +0200
Jiri reported a machine stuck in multi_cpu_stop() with migrate_swap_stop() as function and with the following src,dst cpu
pairs: {11, 4} {13, 11} { 4, 13}
4 11 13
cpuM: queue(4 ,13)
*Ma
cpuN: queue(13,11)
*N Na
*M Mb
cpuO: queue(11, 4)
*O Oa
*Nb
*ObWhere *X denotes the cpu running the queueing of cpu-X and X[ab] denotes the first/second queued work.
You'll observe the top of the workqueue for each cpu: 4,11,13 to be work from cpus: M, O, N resp. IOW. deadlock.
Do away with the queueing trickery and introduce lg_double_lock() to lock both CPUs and fully serialize the stop_two_cpus() callers instead of the partial (and buggy) serialization we have now.
Completely untested..
--- include/linux/lglock.h | 5 +++++ kernel/locking/lglock.c | 22 ++++++++++++++++++++++ kernel/stop_machine.c | 42 +++++------------------------------------- 3 files changed, 32 insertions(+), 37 deletions(-)
include/linux/lglock.h +5 -0
@@ -52,10 +52,15 @@ struct lglock { static struct lglock name = { .lock = &name ## _lock }void lg_lock_init (struct lglock *lg,char *name); +void lg_local_lock (struct lglock *lg);void lg_local_unlock (struct lglock *lg);void lg_local_lock_cpu (struct lglock *lg,int cpu);void lg_local_unlock_cpu (struct lglock *lg,int cpu); + +void lg_double_lock (struct lglock *lg,int cpu1,int cpu2); +void lg_double_unlock (struct lglock *lg,int cpu1,int cpu2); +void lg_global_lock (struct lglock *lg);void lg_global_unlock (struct lglock *lg);
kernel/locking/lglock.c +22 -0
@@ -60,6 +60,28 @@ void lg_local_unlock_cpu(struct lglock *lg, int cpu) }EXPORT_SYMBOL (lg_local_unlock_cpu); +void lg_double_lock (struct lglock *lg,int cpu1,int cpu2) +{ +BUG_ON (cpu1 == cpu2); + +/* lock in cpu order, just like lg_global_lock */ +if (cpu2 < cpu1) +swap (cpu1, cpu2); + +preempt_disable (); +lock_acquire_shared (&lg->lock_dep_map,0 ,0 ,NULL , _RET_IP_); +arch_spin_lock (per_cpu_ptr (lg->lock, cpu1)); +arch_spin_lock (per_cpu_ptr (lg->lock, cpu2)); +} + +void lg_double_unlock (struct lglock *lg,int cpu1,int cpu2) +{ +lock_release (&lg->lock_dep_map,1 , _RET_IP_); +arch_spin_unlock (per_cpu_ptr (lg->lock, cpu1)); +arch_spin_unlock (per_cpu_ptr (lg->lock, cpu2)); +preempt_enable (); +} +void lg_global_lock (struct lglock *lg) {int i;
kernel/stop_machine.c +5 -37
66 lines not on this page · raw message