Skip to contentSkip to conversation

kernel BUG at arch/x86/kvm/mmu.c:655!

DISCUSSION Tomasz Chmielewski 3 participants 8 messages Jan 7, 2011 – Jan 18, 2011

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.

kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0
rmap_remove: ffff88060e9437f8 0 1->BUG

------------[ cut here ]------------
kernel BUG at arch/x86/kvm/mmu.c:655!
invalid opcode: 0000 [#1] SMP 
last sysfs file: /sys/kernel/uevent_seqnum
CPU 0 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 3927, comm: kvm Not tainted 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffffa024261c>]  [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm]
RSP: 0018:ffff88061414bb58  EFLAGS: 00010292
RAX: 000000000000002e RBX: ffff88060e9437f8 RCX: 00000000ffffffff
RDX: 0000000000000000 RSI: 0000000000000082 RDI: 0000000000000246
RBP: ffff88061414bb68 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000003 R12: ffff880520e60000
R13: ffff8804f4514820 R14: ffff880520e60000 R15: 0000000000000000
FS:  00007f049b859730(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f7f1dc2b000 CR3: 00000004f4464000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process kvm (pid: 3927, threadinfo ffff88061414a000, task ffff8806107596d0)
Stack:
 ffff88060e9437f8 00000000000000ff ffff88061414bba8 ffffffffa02427ba
<0> ffff88061414bbb8 ffff8806140fb140 ffff880520e60000 ffff880520e62320
<0> ffff88061414bbb8 0000000000000001 ffff88061414bbe8 ffffffffa024334e
Call Trace:
 [<ffffffffa02427ba>] kvm_mmu_prepare_zap_page+0x8a/0x2f0 [kvm]
 [<ffffffffa024334e>] kvm_mmu_zap_all+0x4e/0x90 [kvm]
 [<ffffffffa0236026>] kvm_arch_flush_shadow+0x16/0x30 [kvm]
 [<ffffffffa022af20>] __kvm_set_memory_region+0x2e0/0x7e0 [kvm]
 [<ffffffffa0239dbd>] ? kvm_set_msr_common+0x4fd/0xd40 [kvm]
 [<ffffffffa024fdc7>] ? pic_unlock+0x27/0xc0 [kvm]
 [<ffffffffa022b461>] kvm_set_memory_region+0x41/0x70 [kvm]
 [<ffffffffa022b4ad>] kvm_vm_ioctl_set_memory_region+0x1d/0x30 [kvm]
 [<ffffffffa022ba7d>] kvm_vm_ioctl+0x3fd/0x440 [kvm]
 [<ffffffff8106c1b0>] ? do_send_sig_info+0x70/0x90
 [<ffffffff8124dd36>] ? security_task_kill+0x16/0x20
 [<ffffffff8114e60f>] do_vfs_ioctl+0x9f/0x550
 [<ffffffff8108db49>] ? sys_futex+0x89/0x160
 [<ffffffff8114eb61>] sys_ioctl+0xa1/0xb0
 [<ffffffff8113e788>] ? sys_read+0x88/0x90
 [<ffffffff8100a332>] system_call_fastpath+0x16/0x1b
Code: a4 48 8b 13 48 89 de 48 c7 c7 70 9f 25 a0 e8 9b b5 32 e1 0f 0b eb fe 48 8b 13 48 89 de 48 c7 c7 90 9f 25 a0 31 c0 e8 83 b5 32 e1 <0f> 0b eb fe 55 48 89 e5 41 55 41 54 53 48 83 ec 08 0f 1f 44 00 
RIP  [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm]
 RSP <ffff88061414bb58>
---[ end trace e0bc209dfb014a9c ]---
BUG: soft lockup - CPU#0 stuck for 67s! [qm:6844]
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
CPU 0 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 6844, comm: qm Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffff810326d4>]  [<ffffffff810326d4>] __ticket_spin_lock+0x14/0x20
RSP: 0000:ffff88014606d7e8  EFLAGS: 00000293
RAX: 000000000000b2b0 RBX: ffff88014606d7e8 RCX: ffffe8ffffc11db8
RDX: 0000000000000000 RSI: ffff880620819500 RDI: ffff880520e60000
RBP: ffffffff8100acce R08: 0000000000000000 R09: ffff8806208197c8
R10: ffff88014606d7a8 R11: 0000000000000000 R12: ffffffffa024271d
R13: ffff88014606d778 R14: ffff880612968000 R15: ffffc90015c48010
FS:  00007f4bafba06e0(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00000000019f4018 CR3: 000000014611d000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process qm (pid: 6844, threadinfo ffff88014606c000, task ffff8806233cdb40)
Stack:
 ffff88014606d7f8 ffffffff81570abe ffff88014606d838 ffffffffa022bf52
<0> ffff880620819500 ffff880520e62448 0000000000000000 ffff880620819500
<0> 00007f0408000000 ffff88014606da18 ffff88014606d868 ffffffff81126d9e
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa022bf52>] ? kvm_mmu_notifier_clear_flush_young+0x42/0x90 [kvm]
 [<ffffffff81126d9e>] ? __mmu_notifier_clear_flush_young+0x4e/0x70
 [<ffffffff811121c1>] ? page_referenced_one+0xd1/0x130
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff81114013>] ? page_referenced+0x143/0x390
 [<ffffffff810f84d6>] ? shrink_active_list+0x1d6/0x370
 [<ffffffff810f8163>] ? shrink_inactive_list+0x233/0x3d0
 [<ffffffff810f8b28>] ? shrink_zone+0x4b8/0x7f0
 [<ffffffff810ed90a>] ? zone_watermark_ok+0x2a/0xf0
 [<ffffffff810f9efb>] ? do_try_to_free_pages+0x9b/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff8110a727>] ? handle_mm_fault+0x667/0xb80
 [<ffffffff8113de19>] ? do_sync_read+0xd9/0x120
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff8124e73a>] ? security_file_permission+0x7a/0x90
 [<ffffffff8113e5f7>] ? vfs_read+0x127/0x180
 [<ffffffff81571415>] ? page_fault+0x25/0x30
Code: c1 d1 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa022bf52>] ? kvm_mmu_notifier_clear_flush_young+0x42/0x90 [kvm]
 [<ffffffff81126d9e>] ? __mmu_notifier_clear_flush_young+0x4e/0x70
 [<ffffffff811121c1>] ? page_referenced_one+0xd1/0x130
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff81114013>] ? page_referenced+0x143/0x390
 [<ffffffff810f84d6>] ? shrink_active_list+0x1d6/0x370
 [<ffffffff810f8163>] ? shrink_inactive_list+0x233/0x3d0
 [<ffffffff810f8b28>] ? shrink_zone+0x4b8/0x7f0
 [<ffffffff810ed90a>] ? zone_watermark_ok+0x2a/0xf0
 [<ffffffff810f9efb>] ? do_try_to_free_pages+0x9b/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff8110a727>] ? handle_mm_fault+0x667/0xb80
 [<ffffffff8113de19>] ? do_sync_read+0xd9/0x120
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff8124e73a>] ? security_file_permission+0x7a/0x90
 [<ffffffff8113e5f7>] ? vfs_read+0x127/0x180
 [<ffffffff81571415>] ? page_fault+0x25/0x30
BUG: soft lockup - CPU#4 stuck for 67s! [kswapd0:60]
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
CPU 4 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 60, comm: kswapd0 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffff810326d6>]  [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20
RSP: 0018:ffff880623413cc0  EFLAGS: 00000297
RAX: 000000000000b1b0 RBX: ffff880623413cc0 RCX: ffffe8ffffc91db8
RDX: 0000000000000000 RSI: ffff880520e60280 RDI: ffff880520e60000
RBP: ffffffff8100acce R08: 0000000000000001 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000c R12: ffffea0014e0cad0
R13: ffffffff810f1a8a R14: ffff880623413c40 R15: ffffea0014e0cd38
FS:  0000000000000000(0000) GS:ffff880001e80000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f32b5b88c40 CR3: 0000000001a2a000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process kswapd0 (pid: 60, threadinfo ffff880623412000, task ffff880624564470)
Stack:
 ffff880623413cd0 ffffffff81570abe ffff880623413d40 ffffffffa024377f
<0> ffff880100000548 ffff880100000540 ffff880623413d00 0000000081570abe
<0> ffff880623413d00 ffff880623413d00 ffff880623413d10 0000000000000000
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa024377f>] ? mmu_shrink+0xaf/0x170 [kvm]
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810f9948>] ? kswapd+0x428/0x940
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff810f9520>] ? kswapd+0x0/0x940
 [<ffffffff810790ee>] ? kthread+0x9e/0xb0
 [<ffffffff8100b124>] ? kernel_thread_helper+0x4/0x10
 [<ffffffff81079050>] ? kthread+0x0/0xb0
 [<ffffffff8100b120>] ? kernel_thread_helper+0x0/0x10
Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa024377f>] ? mmu_shrink+0xaf/0x170 [kvm]
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810f9948>] ? kswapd+0x428/0x940
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff810f9520>] ? kswapd+0x0/0x940
 [<ffffffff810790ee>] ? kthread+0x9e/0xb0
 [<ffffffff8100b124>] ? kernel_thread_helper+0x4/0x10
 [<ffffffff81079050>] ? kthread+0x0/0xb0
 [<ffffffff8100b120>] ? kernel_thread_helper+0x0/0x10
vmbr0: received packet on eth0 with own address as source address
BUG: soft lockup - CPU#2 stuck for 67s! [apache2:6803]
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
CPU 2 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 6803, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffff810326d6>]  [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20
RSP: 0000:ffff8801460bf9b8  EFLAGS: 00000297
RAX: 0000000000007271 RBX: ffff8801460bf9b8 RCX: 0000000000000002
RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88
RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a
R13: ffff8801460bf938 R14: ffff880100000b80 R15: 0000000000000000
FS:  00007fd76a5ef750(0000) GS:ffff880001e40000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000002cabae0 CR3: 00000005b7f27000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process apache2 (pid: 6803, threadinfo ffff8801460be000, task ffff8806236cdb40)
Stack:
 ffff8801460bf9c8 ffffffff81570abe ffff8801460bfa38 ffffffffa02436f8
<0> ffff880100000548 ffff880100000540 ffff8801460bf9f8 ffffffff81570abe
<0> ffff8801460bfa38 ffffffff8118e1db ffff8801460bfa08 0000000000000000
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff8110f438>] ? do_brk+0x278/0x360
 [<ffffffff81571415>] ? page_fault+0x25/0x30
Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff8110f438>] ? do_brk+0x278/0x360
 [<ffffffff81571415>] ? page_fault+0x25/0x30
BUG: soft lockup - CPU#3 stuck for 67s! [apache2:6865]
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
CPU 3 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 6865, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffff810326d4>]  [<ffffffff810326d4>] __ticket_spin_lock+0x14/0x20
RSP: 0000:ffff880621f039b8  EFLAGS: 00000293
RAX: 0000000000007471 RBX: ffff880621f039b8 RCX: 0000000000000002
RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88
RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a
R13: ffff880621f03938 R14: ffff880100000b80 R15: 00000000000006f4
FS:  00007fd76a5ef750(0000) GS:ffff880001e60000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001cf30e0 CR3: 00000001460e3000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process apache2 (pid: 6865, threadinfo ffff880621f02000, task ffff8805da4b5b40)
Stack:
 ffff880621f039c8 ffffffff81570abe ffff880621f03a38 ffffffffa02436f8
<0> ffff880100000548 ffff880100000540 ffff880621f039f8 ffffffff81570abe
<0> ffff880621f03a38 ffffffff8118e1db ffff880621f03a08 0000000000000000
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff81148831>] ? path_put+0x31/0x40
 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150
 [<ffffffff81571415>] ? page_fault+0x25/0x30
Code: c1 d1 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff81148831>] ? path_put+0x31/0x40
 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150
 [<ffffffff81571415>] ? page_fault+0x25/0x30
BUG: soft lockup - CPU#5 stuck for 67s! [apache2:6862]
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
CPU 5 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 6862, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffff810326d6>]  [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20
RSP: 0000:ffff88044c5ab9b8  EFLAGS: 00000293
RAX: 0000000000007371 RBX: ffff88044c5ab9b8 RCX: 0000000000000002
RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88
RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a
R13: ffff88044c5ab938 R14: ffff880100000b80 R15: 0000000000000de8
FS:  00007fd76a5ef750(0000) GS:ffff880001ea0000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000000000282a034 CR3: 000000060eb4b000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process apache2 (pid: 6862, threadinfo ffff88044c5aa000, task ffff8806233616d0)
Stack:
 ffff88044c5ab9c8 ffffffff81570abe ffff88044c5aba38 ffffffffa02436f8
<0> ffff880100000548 ffff880100000540 ffff88044c5ab9f8 ffffffff81570abe
<0> ffff88044c5aba38 ffffffff8118e1db ffff88044c5aba08 0000000000000000
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff810bc3d5>] ? call_rcu_sched+0x15/0x20
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff81148831>] ? path_put+0x31/0x40
 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150
 [<ffffffff81571415>] ? page_fault+0x25/0x30
Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 
Call Trace:
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm]
 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20
 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160
 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190
 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440
 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110
 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890
 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140
 [<ffffffff81109999>] ? do_wp_page+0x209/0x930
 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580
 [<ffffffff810bc3d5>] ? call_rcu_sched+0x15/0x20
 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80
 [<ffffffff81148831>] ? path_put+0x31/0x40
 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660
 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490
 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40
 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150
 [<ffffffff81571415>] ? page_fault+0x25/0x30
Signature · 2 lines
Tomasz Chmielewski
http://wpkg.org
#1 lore raw
Quoted 6 lines · Tomasz Chmielewski

The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.

kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0
rmap_remove: ffff88060e9437f8 0 1->BUG

Is this reproducible?

Was ksm running?

Signature · 1 line
error compiling committee.c: too many arguments to function
#2 lore raw
Quoted 10 lines · Avi Kivity
Quoted 7 lines · Tomasz Chmielewski

The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2).
After a while, the server hanged and was no longer reachable.

kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0
rmap_remove: ffff88060e9437f8 0 1->BUG

Is this reproducible?

Not sure - I can't experiment too much on this server.

What I know is that with 2.6.36.x kernel, this server *always* hangs after a few hours/days, whereas with earlier kernels (2.6.35.x and earlier) it runs fine.

Can't say if the reason is the same or not.

Quoted 1 line

Was ksm running?

No, I don't use ksm.

Signature · 2 lines
Tomasz Chmielewski
http://wpkg.org
#3 lore raw
Quoted 10 lines · Tomasz Chmielewski
Quoted 9 lines · Avi Kivity
Quoted 7 lines · Tomasz Chmielewski

The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2).
After a while, the server hanged and was no longer reachable.

kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0
rmap_remove: ffff88060e9437f8 0 1->BUG

Quoted 8 lines
Quoted 1 line

Is this reproducible?

Not sure - I can't experiment too much on this server.

What I know is that with 2.6.36.x kernel, this server *always* hangs after a few hours/days, whereas with earlier kernels (2.6.35.x and earlier) it runs fine.

Can't say if the reason is the same or not.

Well, whether it's the same bug or not, it's worth fixing.

Signature · 1 line
error compiling committee.c: too many arguments to function
#4 lore raw
In reply to Tomasz Chmielewski
Quoted 40 lines · Tomasz Chmielewski

The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.

kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0
rmap_remove: ffff88060e9437f8 0 1->BUG

------------[ cut here ]------------
kernel BUG at arch/x86/kvm/mmu.c:655!
invalid opcode: 0000 [#1] SMP 
last sysfs file: /sys/kernel/uevent_seqnum
CPU 0 
Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]

Pid: 3927, comm: kvm Not tainted 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522

RIP: 0010:[<ffffffffa024261c>]  [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm]
RSP: 0018:ffff88061414bb58  EFLAGS: 00010292
RAX: 000000000000002e RBX: ffff88060e9437f8 RCX: 00000000ffffffff
RDX: 0000000000000000 RSI: 0000000000000082 RDI: 0000000000000246
RBP: ffff88061414bb68 R08: 0000000000000000 R09: 0000000000000000
R10: 0000000000000001 R11: 0000000000000003 R12: ffff880520e60000
R13: ffff8804f4514820 R14: ffff880520e60000 R15: 0000000000000000
FS:  00007f049b859730(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
CR2: 00007f7f1dc2b000 CR3: 00000004f4464000 CR4: 00000000000026e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process kvm (pid: 3927, threadinfo ffff88061414a000, task ffff8806107596d0)
Stack:
 ffff88060e9437f8 00000000000000ff ffff88061414bba8 ffffffffa02427ba
<0> ffff88061414bbb8 ffff8806140fb140 ffff880520e60000 ffff880520e62320
<0> ffff88061414bbb8 0000000000000001 ffff88061414bbe8 ffffffffa024334e
Call Trace:
 [<ffffffffa02427ba>] kvm_mmu_prepare_zap_page+0x8a/0x2f0 [kvm]
 [<ffffffffa024334e>] kvm_mmu_zap_all+0x4e/0x90 [kvm]
 [<ffffffffa0236026>] kvm_arch_flush_shadow+0x16/0x30 [kvm]
 [<ffffffffa022af20>] __kvm_set_memory_region+0x2e0/0x7e0 [kvm]
 [<ffffffffa0239dbd>] ? kvm_set_msr_common+0x4fd/0xd40 [kvm]
 [<ffffffffa024fdc7>] ? pic_unlock+0x27/0xc0 [kvm]
 [<ffffffffa022b461>] kvm_set_memory_region+0x41/0x70 [kvm]
 [<ffffffffa022b4ad>] kvm_vm_ioctl_set_memory_region+0x1d/0x30 [kvm]
Quoted 8 lines
 [<ffffffffa022ba7d>] kvm_vm_ioctl+0x3fd/0x440 [kvm]
 [<ffffffff8106c1b0>] ? do_send_sig_info+0x70/0x90
 [<ffffffff8124dd36>] ? security_task_kill+0x16/0x20
 [<ffffffff8114e60f>] do_vfs_ioctl+0x9f/0x550
 [<ffffffff8108db49>] ? sys_futex+0x89/0x160
 [<ffffffff8114eb61>] sys_ioctl+0xa1/0xb0
 [<ffffffff8113e788>] ? sys_read+0x88/0x90
 [<ffffffff8100a332>] system_call_fastpath+0x16/0x1b

Tomasz,

This should be fixed by commit eb45fda45f915c7, attached.

commit eb45fda45f915c7ca3e81e005e853cb770da2642

Author: Marcelo Tosatti <[email protected]>
Date:   Mon Oct 25 11:58:22 2010 -0200

KVM: MMU: fix rmap_remove on non present sptes

drop_spte should not attempt to rmap_remove a non present shadow pte.

This fixes a BUG_ON seen on kvm-autotest.

arch/x86/kvm/mmu.c +5 -4
index 908ea54..fb8b376 100644
@@ -720,7 +720,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
 	}
 }
 
-static void set_spte_track_bits(u64 *sptep, u64 new_spte)
+static int set_spte_track_bits(u64 *sptep, u64 new_spte)
 {
 	pfn_t pfn;
 	u64 old_spte = *sptep;
@@ -731,19 +731,20 @@ static void set_spte_track_bits(u64 *sptep, u64 new_spte)
 		old_spte = __xchg_spte(sptep, new_spte);
 
 	if (!is_rmap_spte(old_spte))
-		return;
+		return 0;
 
 	pfn = spte_to_pfn(old_spte);
 	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
 		kvm_set_pfn_accessed(pfn);
 	if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
 		kvm_set_pfn_dirty(pfn);
+	return 1;
 }
 
 static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
 {
-	set_spte_track_bits(sptep, new_spte);
-	rmap_remove(kvm, sptep);
+	if (set_spte_track_bits(sptep, new_spte))
+		rmap_remove(kvm, sptep);
 }
 
 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
#5 lore raw
In reply to Tomasz Chmielewski

Patch against 2.6.36 attached.

commit 9621afb2eb9a1f06d4afcf435175b5323a2b5cc6

Author: Marcelo Tosatti <[email protected]>
Date:   Mon Oct 25 11:58:22 2010 -0200

KVM: MMU: fix rmap_remove on non present sptes

commit eb45fda45f915c7ca3e81e005e853cb770da2642 upstream.

drop_spte should not attempt to rmap_remove a non present shadow pte.

This fixes a BUG_ON seen on kvm-autotest.

arch/x86/kvm/mmu.c +6 -4
index 7fed5b7..cc1bada 100644
@@ -675,7 +675,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
 	}
 }
 
-static void set_spte_track_bits(u64 *sptep, u64 new_spte)
+static int set_spte_track_bits(u64 *sptep, u64 new_spte)
 {
 	pfn_t pfn;
 	u64 old_spte = *sptep;
@@ -687,18 +687,20 @@ static void set_spte_track_bits(u64 *sptep, u64 new_spte)
 		old_spte = __xchg_spte(sptep, new_spte);
 
 	if (!is_rmap_spte(old_spte))
-		return;
+		return 0;
+
 	pfn = spte_to_pfn(old_spte);
 	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
 		kvm_set_pfn_accessed(pfn);
 	if (is_writable_pte(old_spte))
 		kvm_set_pfn_dirty(pfn);
+	return 1;
 }
 
 static void drop_spte(struct kvm *kvm, u64 *sptep, u64 new_spte)
 {
-	set_spte_track_bits(sptep, new_spte);
-	rmap_remove(kvm, sptep);
+	if (set_spte_track_bits(sptep, new_spte))
+		rmap_remove(kvm, sptep);
 }
 
 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
#6 lore raw
Quoted 2 lines · Marcelo Tosatti

Patch against 2.6.36 attached.

Thanks.

Do you know if this bug is present in 2.6.37?

Signature · 2 lines
Tomasz Chmielewski
http://wpkg.org
#7 lore raw
Quoted 7 lines · Tomasz Chmielewski
Quoted 2 lines · Marcelo Tosatti

Patch against 2.6.36 attached.

Thanks.

Do you know if this bug is present in 2.6.37?

Its not.