kernel BUG at arch/x86/kvm/mmu.c:655!
The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.
kvm: 3927: cpu0 unhandled wrmsr: 0x198 data 0
kvm: 3927: cpu1 unhandled wrmsr: 0x198 data 0rmap_remove: ffff88060e9437f8 0 1->BUG
------------[ cut here ]------------ kernel BUG at arch/x86/kvm/mmu.c:655! invalid opcode: 0000 [#1] SMP last sysfs file: /sys/kernel/uevent_seqnum CPU 0 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 3927, comm: kvm Not tainted 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffffa024261c>] [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm] RSP: 0018:ffff88061414bb58 EFLAGS: 00010292 RAX: 000000000000002e RBX: ffff88060e9437f8 RCX: 00000000ffffffff RDX: 0000000000000000 RSI: 0000000000000082 RDI: 0000000000000246 RBP: ffff88061414bb68 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000001 R11: 0000000000000003 R12: ffff880520e60000 R13: ffff8804f4514820 R14: ffff880520e60000 R15: 0000000000000000 FS: 00007f049b859730(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 00007f7f1dc2b000 CR3: 00000004f4464000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process kvm (pid: 3927, threadinfo ffff88061414a000, task ffff8806107596d0) Stack: ffff88060e9437f8 00000000000000ff ffff88061414bba8 ffffffffa02427ba <0> ffff88061414bbb8 ffff8806140fb140 ffff880520e60000 ffff880520e62320 <0> ffff88061414bbb8 0000000000000001 ffff88061414bbe8 ffffffffa024334e Call Trace: [<ffffffffa02427ba>] kvm_mmu_prepare_zap_page+0x8a/0x2f0 [kvm] [<ffffffffa024334e>] kvm_mmu_zap_all+0x4e/0x90 [kvm] [<ffffffffa0236026>] kvm_arch_flush_shadow+0x16/0x30 [kvm] [<ffffffffa022af20>] __kvm_set_memory_region+0x2e0/0x7e0 [kvm] [<ffffffffa0239dbd>] ? kvm_set_msr_common+0x4fd/0xd40 [kvm] [<ffffffffa024fdc7>] ? pic_unlock+0x27/0xc0 [kvm] [<ffffffffa022b461>] kvm_set_memory_region+0x41/0x70 [kvm] [<ffffffffa022b4ad>] kvm_vm_ioctl_set_memory_region+0x1d/0x30 [kvm] [<ffffffffa022ba7d>] kvm_vm_ioctl+0x3fd/0x440 [kvm] [<ffffffff8106c1b0>] ? do_send_sig_info+0x70/0x90 [<ffffffff8124dd36>] ? security_task_kill+0x16/0x20 [<ffffffff8114e60f>] do_vfs_ioctl+0x9f/0x550 [<ffffffff8108db49>] ? sys_futex+0x89/0x160 [<ffffffff8114eb61>] sys_ioctl+0xa1/0xb0 [<ffffffff8113e788>] ? sys_read+0x88/0x90 [<ffffffff8100a332>] system_call_fastpath+0x16/0x1b Code: a4 48 8b 13 48 89 de 48 c7 c7 70 9f 25 a0 e8 9b b5 32 e1 0f 0b eb fe 48 8b 13 48 89 de 48 c7 c7 90 9f 25 a0 31 c0 e8 83 b5 32 e1 <0f> 0b eb fe 55 48 89 e5 41 55 41 54 53 48 83 ec 08 0f 1f 44 00 RIP [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm] RSP <ffff88061414bb58> ---[ end trace e0bc209dfb014a9c ]--- BUG: soft lockup - CPU#0 stuck for 67s! [qm:6844] Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan] CPU 0 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 6844, comm: qm Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffff810326d4>] [<ffffffff810326d4>] __ticket_spin_lock+0x14/0x20 RSP: 0000:ffff88014606d7e8 EFLAGS: 00000293 RAX: 000000000000b2b0 RBX: ffff88014606d7e8 RCX: ffffe8ffffc11db8 RDX: 0000000000000000 RSI: ffff880620819500 RDI: ffff880520e60000 RBP: ffffffff8100acce R08: 0000000000000000 R09: ffff8806208197c8 R10: ffff88014606d7a8 R11: 0000000000000000 R12: ffffffffa024271d R13: ffff88014606d778 R14: ffff880612968000 R15: ffffc90015c48010 FS: 00007f4bafba06e0(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00000000019f4018 CR3: 000000014611d000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process qm (pid: 6844, threadinfo ffff88014606c000, task ffff8806233cdb40) Stack: ffff88014606d7f8 ffffffff81570abe ffff88014606d838 ffffffffa022bf52 <0> ffff880620819500 ffff880520e62448 0000000000000000 ffff880620819500 <0> 00007f0408000000 ffff88014606da18 ffff88014606d868 ffffffff81126d9e Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa022bf52>] ? kvm_mmu_notifier_clear_flush_young+0x42/0x90 [kvm] [<ffffffff81126d9e>] ? __mmu_notifier_clear_flush_young+0x4e/0x70 [<ffffffff811121c1>] ? page_referenced_one+0xd1/0x130 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff81114013>] ? page_referenced+0x143/0x390 [<ffffffff810f84d6>] ? shrink_active_list+0x1d6/0x370 [<ffffffff810f8163>] ? shrink_inactive_list+0x233/0x3d0 [<ffffffff810f8b28>] ? shrink_zone+0x4b8/0x7f0 [<ffffffff810ed90a>] ? zone_watermark_ok+0x2a/0xf0 [<ffffffff810f9efb>] ? do_try_to_free_pages+0x9b/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff8110a727>] ? handle_mm_fault+0x667/0xb80 [<ffffffff8113de19>] ? do_sync_read+0xd9/0x120 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff8124e73a>] ? security_file_permission+0x7a/0x90 [<ffffffff8113e5f7>] ? vfs_read+0x127/0x180 [<ffffffff81571415>] ? page_fault+0x25/0x30 Code: c1 d1 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa022bf52>] ? kvm_mmu_notifier_clear_flush_young+0x42/0x90 [kvm] [<ffffffff81126d9e>] ? __mmu_notifier_clear_flush_young+0x4e/0x70 [<ffffffff811121c1>] ? page_referenced_one+0xd1/0x130 [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff81114013>] ? page_referenced+0x143/0x390 [<ffffffff810f84d6>] ? shrink_active_list+0x1d6/0x370 [<ffffffff810f8163>] ? shrink_inactive_list+0x233/0x3d0 [<ffffffff810f8b28>] ? shrink_zone+0x4b8/0x7f0 [<ffffffff810ed90a>] ? zone_watermark_ok+0x2a/0xf0 [<ffffffff810f9efb>] ? do_try_to_free_pages+0x9b/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff8110a727>] ? handle_mm_fault+0x667/0xb80 [<ffffffff8113de19>] ? do_sync_read+0xd9/0x120 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff8124e73a>] ? security_file_permission+0x7a/0x90 [<ffffffff8113e5f7>] ? vfs_read+0x127/0x180 [<ffffffff81571415>] ? page_fault+0x25/0x30 BUG: soft lockup - CPU#4 stuck for 67s! [kswapd0:60] Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan] CPU 4 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 60, comm: kswapd0 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffff810326d6>] [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20 RSP: 0018:ffff880623413cc0 EFLAGS: 00000297 RAX: 000000000000b1b0 RBX: ffff880623413cc0 RCX: ffffe8ffffc91db8 RDX: 0000000000000000 RSI: ffff880520e60280 RDI: ffff880520e60000 RBP: ffffffff8100acce R08: 0000000000000001 R09: 0000000000000000 R10: 0000000000000000 R11: 000000000000000c R12: ffffea0014e0cad0 R13: ffffffff810f1a8a R14: ffff880623413c40 R15: ffffea0014e0cd38 FS: 0000000000000000(0000) GS:ffff880001e80000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 00007f32b5b88c40 CR3: 0000000001a2a000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process kswapd0 (pid: 60, threadinfo ffff880623412000, task ffff880624564470) Stack: ffff880623413cd0 ffffffff81570abe ffff880623413d40 ffffffffa024377f <0> ffff880100000548 ffff880100000540 ffff880623413d00 0000000081570abe <0> ffff880623413d00 ffff880623413d00 ffff880623413d10 0000000000000000 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa024377f>] ? mmu_shrink+0xaf/0x170 [kvm] [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810f9948>] ? kswapd+0x428/0x940 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff810f9520>] ? kswapd+0x0/0x940 [<ffffffff810790ee>] ? kthread+0x9e/0xb0 [<ffffffff8100b124>] ? kernel_thread_helper+0x4/0x10 [<ffffffff81079050>] ? kthread+0x0/0xb0 [<ffffffff8100b120>] ? kernel_thread_helper+0x0/0x10 Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa024377f>] ? mmu_shrink+0xaf/0x170 [kvm] [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810f9948>] ? kswapd+0x428/0x940 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff810f9520>] ? kswapd+0x0/0x940 [<ffffffff810790ee>] ? kthread+0x9e/0xb0 [<ffffffff8100b124>] ? kernel_thread_helper+0x4/0x10 [<ffffffff81079050>] ? kthread+0x0/0xb0 [<ffffffff8100b120>] ? kernel_thread_helper+0x0/0x10 vmbr0: received packet on eth0 with own address as source address BUG: soft lockup - CPU#2 stuck for 67s! [apache2:6803] Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan] CPU 2 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 6803, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffff810326d6>] [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20 RSP: 0000:ffff8801460bf9b8 EFLAGS: 00000297 RAX: 0000000000007271 RBX: ffff8801460bf9b8 RCX: 0000000000000002 RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88 RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a R13: ffff8801460bf938 R14: ffff880100000b80 R15: 0000000000000000 FS: 00007fd76a5ef750(0000) GS:ffff880001e40000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000002cabae0 CR3: 00000005b7f27000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process apache2 (pid: 6803, threadinfo ffff8801460be000, task ffff8806236cdb40) Stack: ffff8801460bf9c8 ffffffff81570abe ffff8801460bfa38 ffffffffa02436f8 <0> ffff880100000548 ffff880100000540 ffff8801460bf9f8 ffffffff81570abe <0> ffff8801460bfa38 ffffffff8118e1db ffff8801460bfa08 0000000000000000 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff8110f438>] ? do_brk+0x278/0x360 [<ffffffff81571415>] ? page_fault+0x25/0x30 Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff8110f438>] ? do_brk+0x278/0x360 [<ffffffff81571415>] ? page_fault+0x25/0x30 BUG: soft lockup - CPU#3 stuck for 67s! [apache2:6865] Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan] CPU 3 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 6865, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffff810326d4>] [<ffffffff810326d4>] __ticket_spin_lock+0x14/0x20 RSP: 0000:ffff880621f039b8 EFLAGS: 00000293 RAX: 0000000000007471 RBX: ffff880621f039b8 RCX: 0000000000000002 RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88 RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a R13: ffff880621f03938 R14: ffff880100000b80 R15: 00000000000006f4 FS: 00007fd76a5ef750(0000) GS:ffff880001e60000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000001cf30e0 CR3: 00000001460e3000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process apache2 (pid: 6865, threadinfo ffff880621f02000, task ffff8805da4b5b40) Stack: ffff880621f039c8 ffffffff81570abe ffff880621f03a38 ffffffffa02436f8 <0> ffff880100000548 ffff880100000540 ffff880621f039f8 ffffffff81570abe <0> ffff880621f03a38 ffffffff8118e1db ffff880621f03a08 0000000000000000 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff81148831>] ? path_put+0x31/0x40 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150 [<ffffffff81571415>] ? page_fault+0x25/0x30 Code: c1 d1 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 <8a> 07 eb f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff81148831>] ? path_put+0x31/0x40 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150 [<ffffffff81571415>] ? page_fault+0x25/0x30 BUG: soft lockup - CPU#5 stuck for 67s! [apache2:6862] Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan] CPU 5 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]
Pid: 6862, comm: apache2 Tainted: G D 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffff810326d6>] [<ffffffff810326d6>] __ticket_spin_lock+0x16/0x20 RSP: 0000:ffff88044c5ab9b8 EFLAGS: 00000293 RAX: 0000000000007371 RBX: ffff88044c5ab9b8 RCX: 0000000000000002 RDX: 00000000000200da RSI: 0000000000000000 RDI: ffffffffa0261a88 RBP: ffffffff8100acce R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 000000000000000c R12: ffffffff810f1a8a R13: ffff88044c5ab938 R14: ffff880100000b80 R15: 0000000000000de8 FS: 00007fd76a5ef750(0000) GS:ffff880001ea0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000000000282a034 CR3: 000000060eb4b000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process apache2 (pid: 6862, threadinfo ffff88044c5aa000, task ffff8806233616d0) Stack: ffff88044c5ab9c8 ffffffff81570abe ffff88044c5aba38 ffffffffa02436f8 <0> ffff880100000548 ffff880100000540 ffff88044c5ab9f8 ffffffff81570abe <0> ffff88044c5aba38 ffffffff8118e1db ffff88044c5aba08 0000000000000000 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff810bc3d5>] ? call_rcu_sched+0x15/0x20 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff81148831>] ? path_put+0x31/0x40 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150 [<ffffffff81571415>] ? page_fault+0x25/0x30 Code: 24 03 81 e9 dd fe ff ff 90 90 90 90 90 90 90 90 90 90 90 90 90 55 b8 00 01 00 00 48 89 e5 f0 66 0f c1 07 38 e0 74 06 f3 90 8a 07 <eb> f6 c9 c3 66 0f 1f 44 00 00 55 48 89 e5 0f b7 07 38 e0 8d 90 Call Trace: [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffffa02436f8>] ? mmu_shrink+0x28/0x170 [kvm] [<ffffffff81570abe>] ? _raw_spin_lock+0xe/0x20 [<ffffffff8118e1db>] ? mb_cache_shrink_fn+0xfb/0x160 [<ffffffff810f8ffd>] ? shrink_slab+0x8d/0x190 [<ffffffff810fa15c>] ? do_try_to_free_pages+0x2fc/0x440 [<ffffffff810fa52a>] ? try_to_free_pages+0xba/0x110 [<ffffffff810f0463>] ? __alloc_pages_nodemask+0x473/0x890 [<ffffffff811256f9>] ? alloc_page_vma+0x89/0x140 [<ffffffff81109999>] ? do_wp_page+0x209/0x930 [<ffffffff81105f3d>] ? __do_fault+0x47d/0x580 [<ffffffff810bc3d5>] ? call_rcu_sched+0x15/0x20 [<ffffffff8110a6fd>] ? handle_mm_fault+0x63d/0xb80 [<ffffffff81148831>] ? path_put+0x31/0x40 [<ffffffff8114cf70>] ? do_filp_open+0x250/0x660 [<ffffffff815748ab>] ? do_page_fault+0x1eb/0x490 [<ffffffff81079670>] ? autoremove_wake_function+0x0/0x40 [<ffffffff81157f89>] ? alloc_fd+0x129/0x150 [<ffffffff81571415>] ? page_fault+0x25/0x30
Signature · 2 lines
Tomasz Chmielewski http://wpkg.org
Quoted 6 lines · Tomasz Chmielewski
The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.
kvm: 3927: cpu0 unhandled wrmsr:
0x198data 0
kvm: 3927: cpu1 unhandled wrmsr:0x198data 0rmap_remove:ffff88060e9437f80 1->BUG
Is this reproducible?
Was ksm running?
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 10 lines · Avi Kivity
Quoted 7 lines · Tomasz Chmielewski
The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2).
After a while, the server hanged and was no longer reachable.kvm: 3927: cpu0 unhandled wrmsr:
0x198data 0
kvm: 3927: cpu1 unhandled wrmsr:0x198data 0rmap_remove:ffff88060e9437f80 1->BUGIs this reproducible?
Not sure - I can't experiment too much on this server.
What I know is that with 2.6.36.x kernel, this server *always* hangs after a few hours/days, whereas with earlier kernels (2.6.35.x and earlier) it runs fine.
Can't say if the reason is the same or not.
Quoted 1 line
Was ksm running?
No, I don't use ksm.
Signature · 2 lines
Tomasz Chmielewski http://wpkg.org
Quoted 10 lines · Tomasz Chmielewski
Quoted 9 lines · Avi Kivity
Quoted 7 lines · Tomasz Chmielewski
The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2).
After a while, the server hanged and was no longer reachable.kvm: 3927: cpu0 unhandled wrmsr:
0x198data 0
kvm: 3927: cpu1 unhandled wrmsr:0x198data 0rmap_remove:ffff88060e9437f80 1->BUG
Quoted 8 lines
Quoted 1 line
Is this reproducible?
Not sure - I can't experiment too much on this server.
What I know is that with 2.6.36.x kernel, this server *always* hangs after a few hours/days, whereas with earlier kernels (2.6.35.x and earlier) it runs fine.
Can't say if the reason is the same or not.
Well, whether it's the same bug or not, it's worth fixing.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 40 lines · Tomasz Chmielewski
The following happened when I tried to reboot a virtual machine (host running qemu 0.13.0, kernel 2.6.36.2). After a while, the server hanged and was no longer reachable.
kvm: 3927: cpu0 unhandled wrmsr:
0x198data 0
kvm: 3927: cpu1 unhandled wrmsr:0x198data 0rmap_remove:ffff88060e9437f80 1->BUG------------[ cut here ]------------ kernel BUG at arch/x86/kvm/mmu.c:655! invalid opcode: 0000 [#1] SMP last sysfs file: /sys/kernel/uevent_seqnum CPU 0 Modules linked in: ipt_MASQUERADE vhost_net kvm_intel kvm iptable_filter xt_tcpudp iptable_nat nf_nat nf_conntrack_ipv4 nf_conntrack nf_defrag_ipv4 ip_tables x_tables bridge stp coretemp f71882fg snd_pcm snd_timer tpm_tis snd tpm soundcore tpm_bios snd_page_alloc i2c_i801 i7core_edac pcspkr edac_core shpchp r8169 mii raid10 raid456 async_pq async_xor xor async_memcpy async_raid6_recov raid6_pq async_tx raid1 raid0 ahci libahci sata_nv sata_sil sata_via 3w_9xxx 3w_xxxx [last unloaded: scsi_wait_scan]Pid: 3927, comm: kvm Not tainted 2.6.36.2 #1 MSI X58 Pro-E (MS-7522)/MS-7522
RIP: 0010:[<ffffffffa024261c>] [<ffffffffa024261c>] drop_spte+0x1ec/0x1f0 [kvm] RSP: 0018:ffff88061414bb58 EFLAGS: 00010292 RAX: 000000000000002e RBX: ffff88060e9437f8 RCX: 00000000ffffffff RDX: 0000000000000000 RSI: 0000000000000082 RDI: 0000000000000246 RBP: ffff88061414bb68 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000001 R11: 0000000000000003 R12: ffff880520e60000 R13: ffff8804f4514820 R14: ffff880520e60000 R15: 0000000000000000 FS: 00007f049b859730(0000) GS:ffff880001e00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b CR2: 00007f7f1dc2b000 CR3: 00000004f4464000 CR4: 00000000000026e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 Process kvm (pid: 3927, threadinfo ffff88061414a000, task ffff8806107596d0) Stack: ffff88060e9437f8 00000000000000ff ffff88061414bba8 ffffffffa02427ba <0> ffff88061414bbb8 ffff8806140fb140 ffff880520e60000 ffff880520e62320 <0> ffff88061414bbb8 0000000000000001 ffff88061414bbe8 ffffffffa024334e Call Trace: [<ffffffffa02427ba>] kvm_mmu_prepare_zap_page+0x8a/0x2f0 [kvm] [<ffffffffa024334e>] kvm_mmu_zap_all+0x4e/0x90 [kvm] [<ffffffffa0236026>] kvm_arch_flush_shadow+0x16/0x30 [kvm] [<ffffffffa022af20>] __kvm_set_memory_region+0x2e0/0x7e0 [kvm] [<ffffffffa0239dbd>] ? kvm_set_msr_common+0x4fd/0xd40 [kvm] [<ffffffffa024fdc7>] ? pic_unlock+0x27/0xc0 [kvm] [<ffffffffa022b461>] kvm_set_memory_region+0x41/0x70 [kvm] [<ffffffffa022b4ad>] kvm_vm_ioctl_set_memory_region+0x1d/0x30 [kvm]
Quoted 8 lines
[<ffffffffa022ba7d>] kvm_vm_ioctl+0x3fd/0x440 [kvm] [<ffffffff8106c1b0>] ? do_send_sig_info+0x70/0x90 [<ffffffff8124dd36>] ? security_task_kill+0x16/0x20 [<ffffffff8114e60f>] do_vfs_ioctl+0x9f/0x550 [<ffffffff8108db49>] ? sys_futex+0x89/0x160 [<ffffffff8114eb61>] sys_ioctl+0xa1/0xb0 [<ffffffff8113e788>] ? sys_read+0x88/0x90 [<ffffffff8100a332>] system_call_fastpath+0x16/0x1b
Tomasz,
This should be fixed by commit eb45fda45f915c7, attached.
commit eb45fda45f915c7ca3e81e005e853cb770da2642
Author: Marcelo Tosatti <[email protected]> Date: Mon Oct 25 11:58:22 2010 -0200
KVM: MMU: fix rmap_remove on non present sptes
drop_spte should not attempt to rmap_remove a non present shadow pte.
This fixes a BUG_ON seen on kvm-autotest.
arch/x86/kvm/mmu.c +5 -4
@@ -720,7 +720,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void set_spte_track_bits (u64 *sptep,u64 new_spte) +static int set_spte_track_bits (u64 *sptep,u64 new_spte) {pfn_t pfn;u64 old_spte = *sptep;
@@ -731,19 +731,20 @@ static void set_spte_track_bits(u64 *sptep, u64 new_spte) old_spte =__xchg_spte (sptep, new_spte);if (!is_rmap_spte (old_spte)) -return ; +return 0 ; pfn =spte_to_pfn (old_spte);if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)kvm_set_pfn_accessed (pfn);if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))kvm_set_pfn_dirty (pfn); +return 1 ; }static void drop_spte (struct kvm *kvm,u64 *sptep,u64 new_spte) { -set_spte_track_bits (sptep, new_spte); -rmap_remove (kvm, sptep); +if (set_spte_track_bits (sptep, new_spte)) +rmap_remove (kvm, sptep); }static u64 *rmap_next (struct kvm *kvm,unsigned long *rmapp,u64 *spte)
Patch against 2.6.36 attached.
commit 9621afb2eb9a1f06d4afcf435175b5323a2b5cc6
Author: Marcelo Tosatti <[email protected]> Date: Mon Oct 25 11:58:22 2010 -0200
KVM: MMU: fix rmap_remove on non present sptes
commit eb45fda45f915c7ca3e81e005e853cb770da2642 upstream.
drop_spte should not attempt to rmap_remove a non present shadow pte.
This fixes a BUG_ON seen on kvm-autotest.
arch/x86/kvm/mmu.c +6 -4
@@ -675,7 +675,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void set_spte_track_bits (u64 *sptep,u64 new_spte) +static int set_spte_track_bits (u64 *sptep,u64 new_spte) {pfn_t pfn;u64 old_spte = *sptep;
@@ -687,18 +687,20 @@ static void set_spte_track_bits(u64 *sptep, u64 new_spte) old_spte =__xchg_spte (sptep, new_spte);if (!is_rmap_spte (old_spte)) -return ; +return 0 ; + pfn =spte_to_pfn (old_spte);if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)kvm_set_pfn_accessed (pfn);if (is_writable_pte (old_spte))kvm_set_pfn_dirty (pfn); +return 1 ; }static void drop_spte (struct kvm *kvm,u64 *sptep,u64 new_spte) { -set_spte_track_bits (sptep, new_spte); -rmap_remove (kvm, sptep); +if (set_spte_track_bits (sptep, new_spte)) +rmap_remove (kvm, sptep); }static u64 *rmap_next (struct kvm *kvm,unsigned long *rmapp,u64 *spte)
Quoted 2 lines · Marcelo Tosatti
Patch against 2.6.36 attached.
Thanks.
Do you know if this bug is present in 2.6.37?
Signature · 2 lines
Tomasz Chmielewski http://wpkg.org
Quoted 7 lines · Tomasz Chmielewski
Quoted 2 lines · Marcelo Tosatti
Patch against 2.6.36 attached.
Thanks.
Do you know if this bug is present in 2.6.37?
Its not.